Skip to content

Allow dynamodb to run initial sync only #361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adiom-mark merged 2 commits into from
Jan 28, 2026
Merged

Allow dynamodb to run initial sync only #361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8d221f1
Allow dynamodb to run initial sync only
Jan 10, 2026
d0546b6
check for throttled
Jan 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 97 additions & 28 deletions connectors/dynamodb/conn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"errors"
"fmt"
"log/slog"
"math/rand/v2"
"os"
"sync"

"connectrpc.com/connect"
Expand All @@ -16,6 +18,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/aws/aws-sdk-go-v2/service/dynamodbstreams"
"golang.org/x/sync/errgroup"
)
Expand Down Expand Up @@ -71,36 +74,48 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene
eg.Go(func() error {
tableDetails, err := c.client.TableDetails(egCtx, name)
if err != nil {
if errors.Is(err, ErrNotFound) {
slog.Warn("Table not found. Ignoring.", "name", name)
return nil
}
Comment on lines +77 to +80
Copy link

@coderabbitai coderabbitai bot Jan 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Avoid silently skipping missing tables when namespaces are explicit.

If the user specified namespaces, ignoring ErrNotFound can lead to silent data loss. Consider only skipping when namespaces were auto-discovered.

🔧 Suggested fix 
-			if err != nil {
-				if errors.Is(err, ErrNotFound) {
-					slog.Warn("Table not found. Ignoring.", "name", name)
-					return nil
-				}
-				return err
-			}
+			if err != nil {
+				if errors.Is(err, ErrNotFound) {
+					if len(namespaces) == 0 {
+						slog.Warn("Table not found. Ignoring.", "name", name)
+						return nil
+					}
+				}
+				return err
+			}
🤖 Prompt for AI Agents 
In `@connectors/dynamodb/conn.go` around lines 77 - 80, When handling ErrNotFound
for a table (errors.Is(err, ErrNotFound) with the "name" variable), do not
silently return nil when the user provided explicit namespaces; only ignore
missing tables when namespaces were auto-discovered. Modify the branch to check
the namespace source (e.g., a boolean like namespacesExplicit or a config field
that indicates user-specified namespaces) and: if the table is missing and
namespaces were auto-discovered, log the warning and return nil; if namespaces
were explicitly provided, return the ErrNotFound (or propagate err) instead of
nil so the caller can handle the missing table.

return err
}

if tableDetails.StreamARN == "" {
if c.spec == "localstack" {
slog.Debug("No stream found, starting stream", "table", name)
_, err := c.client.StartStream(egCtx, name, false)
if err != nil {
return err
if r.Msg.GetUpdates() {
if tableDetails.StreamARN == "" {
if c.spec == "localstack" {
slog.Debug("No stream found, starting stream", "table", name)
streamARN, err := c.client.StartStream(egCtx, name, false)
if err != nil {
return err
}
tableDetails.StreamARN = streamARN
} else {
return fmt.Errorf("no stream found")
}
} else {
return fmt.Errorf("no stream found")
}
} else if tableDetails.IncompatibleStream {
if c.spec == "localstack" {
slog.Debug("Incompatible stream found, restarting stream", "table", name)
_, err := c.client.StartStream(egCtx, name, true)
if err != nil {
return err
} else if tableDetails.IncompatibleStream {
if c.spec == "localstack" {
slog.Debug("Incompatible stream found, restarting stream", "table", name)
streamARN, err := c.client.StartStream(egCtx, name, true)
if err != nil {
return err
}
tableDetails.StreamARN = streamARN
} else {
return fmt.Errorf("incompatible stream found")
}
} else {
return fmt.Errorf("incompatible stream found")
}

state, err := c.client.GetStreamState(egCtx, tableDetails.StreamARN)
if err != nil {
return err
}
statesCh <- state
}

state, err := c.client.GetStreamState(ctx, tableDetails.StreamARN)
if err != nil {
return err
if !r.Msg.GetInitialSync() {
return nil
}
statesCh <- state

// TODO: reconsider how to map namespaces properly
ns := name
Expand Down Expand Up @@ -139,22 +154,34 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene
return nil, connect.NewError(connect.CodeInternal, err)
}

var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err = enc.Encode(stateMap)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, err)
var updates []*adiomv1.UpdatesPartition
if r.Msg.GetUpdates() {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err = enc.Encode(stateMap)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, err)
}
updates = append(updates, &adiomv1.UpdatesPartition{Namespaces: namespaces, Cursor: buf.Bytes()})
}

if len(partitions) > 0 {
slog.Debug("shuffling partitions")
rand.Shuffle(len(partitions), func(i, j int) {
partitions[i], partitions[j] = partitions[j], partitions[i]
})
}

return connect.NewResponse(&adiomv1.GeneratePlanResponse{
Partitions: partitions,
UpdatesPartitions: []*adiomv1.UpdatesPartition{{Namespaces: namespaces, Cursor: buf.Bytes()}},
UpdatesPartitions: updates,
}), nil
}

// GetInfo implements adiomv1connect.ConnectorServiceHandler.
func (c *conn) GetInfo(context.Context, *connect.Request[adiomv1.GetInfoRequest]) (*connect.Response[adiomv1.GetInfoResponse], error) {
return connect.NewResponse(&adiomv1.GetInfoResponse{
Id: c.options.ID,
DbType: "dynamodb",
Version: "",
Spec: c.spec,
Expand Down Expand Up @@ -185,6 +212,23 @@ func (c *conn) GetNamespaceMetadata(ctx context.Context, r *connect.Request[adio
}), nil
}

func isThrottled(err error) bool {
var provisionedThroughputExceededException *types.ProvisionedThroughputExceededException
var throttlingException *types.ThrottlingException
var requestLimitExceeded *types.RequestLimitExceeded

if errors.As(err, &provisionedThroughputExceededException) {
return true
}
if errors.As(err, &throttlingException) {
return true
}
if errors.As(err, &requestLimitExceeded) {
return true
}
return false
}

// ListData implements adiomv1connect.ConnectorServiceHandler.
func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListDataRequest]) (*connect.Response[adiomv1.ListDataResponse], error) {
cursor := r.Msg.GetCursor()
Expand All @@ -197,6 +241,9 @@ func (c *conn) ListData(ctx context.Context, r *connect.Request[adiomv1.ListData
if errors.Is(err, ErrNotFound) {
return nil, connect.NewError(connect.CodeNotFound, err)
}
if isThrottled(err) {
return nil, connect.NewError(connect.CodeResourceExhausted, err)
}
return nil, connect.NewError(connect.CodeInternal, err)
}

Expand Down Expand Up @@ -355,7 +402,10 @@ func (c *conn) WriteUpdates(context.Context, *connect.Request[adiomv1.WriteUpdat
func AWSClientHelper(connStr string) (*dynamodb.Client, *dynamodbstreams.Client) {
var endpoint string
if connStr == "localstack" {
endpoint = "http://localhost:4566"
endpoint = os.Getenv("AWS_ENDPOINT_URL")
if endpoint == "" {
endpoint = "http://localhost:4566"
}
}
awsConfig, err := config.LoadDefaultConfig(context.Background())
if err != nil {
Expand All @@ -375,10 +425,29 @@ func AWSClientHelper(connStr string) (*dynamodb.Client, *dynamodbstreams.Client)
}

type Options struct {
ID string
DocsPerSegment int
PlanParallelism int
}

func WithID(s string) func(*Options) {
return func(o *Options) {
o.ID = s
}
}

func WithPlanParallelism(n int) func(*Options) {
return func(o *Options) {
o.PlanParallelism = n
}
}

func WithDocsPerSegment(n int) func(*Options) {
return func(o *Options) {
o.DocsPerSegment = n
}
}
Comment on lines +428 to +449
Copy link

@coderabbitai coderabbitai bot Jan 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

🏁 Script executed:

# First, find the file and examine the context around the WithPlanParallelism function
fd -t f "conn.go" | grep dynamodb

Repository: adiom-data/dsync

Length of output: 87

🏁 Script executed:

# Search for where PlanParallelism is used in the codebase
rg "PlanParallelism" --type go -B 3 -A 3

Repository: adiom-data/dsync

Length of output: 2523

🏁 Script executed:

# Search for errgroup.SetLimit calls to understand how PlanParallelism is used
rg "SetLimit" --type go -B 5 -A 5

Repository: adiom-data/dsync

Length of output: 4530

🌐 Web query:

golang.org/x/sync/errgroup SetLimit panic behavior zero negative values

💡 Result:

  • SetLimit(n) does not panic just because n is 0 or negative.

    • n < 0: disables the limit (“no limit”) by setting the internal semaphore to nil. [1], [2]
    • n == 0: sets a limit of zero (“prevent any new goroutines from being added”). Practically, Group.Go(...) will block forever trying to acquire the limit, and TryGo(...) will always return false (it can never acquire a slot). [1], [2]

  • When it does panic: SetLimit panics if you call it while the group still has active goroutines (i.e., the limit is being modified while tokens are currently held). The implementation checks len(g.sem) != 0 and panics in that case. [2]

Sources:
[1] golang.org/x/sync/errgroup docs on pkg.go.dev (SetLimit behavior for negative/zero, and “must not be modified while any goroutines … are active”). (pkg.go.dev)
[2] Go codereview showing the SetLimit implementation and its panic condition. (groups.google.com)

Citations:

Guard PlanParallelism to prevent deadlock or unintended unlimited concurrency.

While errgroup.SetLimit does not panic on zero or negative values, passing these values causes problematic behavior: n == 0 causes Go() calls to block indefinitely, and n < 0 disables the limit entirely. Clamp the value to a minimum of 1.

♻️ Suggested guard 
 func WithPlanParallelism(n int) func(*Options) {
 	return func(o *Options) {
-		o.PlanParallelism = n
+		if n < 1 {
+			n = 1
+		}
+		o.PlanParallelism = n
 	}
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
ID string
DocsPerSegment int
PlanParallelism int
}
func WithID(s string) func(*Options) {
return func(o *Options) {
o.ID = s
}
}
func WithPlanParallelism(n int) func(*Options) {
return func(o *Options) {
o.PlanParallelism = n
}
}
func WithDocsPerSegment(n int) func(*Options) {
return func(o *Options) {
o.DocsPerSegment = n
}
}
ID string
DocsPerSegment int
PlanParallelism int
}
func WithID(s string) func(*Options) {
return func(o *Options) {
o.ID = s
}
}
func WithPlanParallelism(n int) func(*Options) {
return func(o *Options) {
if n < 1 {
n = 1
}
o.PlanParallelism = n
}
}
func WithDocsPerSegment(n int) func(*Options) {
return func(o *Options) {
o.DocsPerSegment = n
}
}
🤖 Prompt for AI Agents 
In `@connectors/dynamodb/conn.go` around lines 420 - 441, The WithPlanParallelism
setter currently assigns the raw integer to Options.PlanParallelism which can be
zero or negative and lead to deadlocks or disabled limits when later passed to
errgroup.SetLimit; update the WithPlanParallelism function to clamp the input to
a minimum of 1 before assigning (e.g., if n < 1 then set n = 1) so
Options.PlanParallelism is always >= 1 and downstream calls like
errgroup.SetLimit(Options.PlanParallelism) are safe.


func NewConn(connStr string, optFns ...func(*Options)) adiomv1connect.ConnectorServiceHandler {
opts := Options{
DocsPerSegment: 50000,
Expand Down
17 changes: 12 additions & 5 deletions connectors/dynamodb/conv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ func fromBson(bs interface{}) (types.AttributeValue, error) {
return &types.AttributeValueMemberB{Value: b.Data}, nil
case primitive.Decimal128:
return &types.AttributeValueMemberN{Value: b.String()}, nil
case nil:
return &types.AttributeValueMemberNULL{Value: true}, nil
default:
return &types.AttributeValueMemberS{Value: "XUnsupportedX"}, nil
}
Expand Down Expand Up @@ -167,6 +169,9 @@ func toBson(av types.AttributeValue) (interface{}, error) {
}
return arr, nil

case *types.AttributeValueMemberNULL:
return nil, nil

default:
return nil, fmt.Errorf("unknown attribute %T", av)
}
Expand Down Expand Up @@ -203,11 +208,11 @@ func itemsToBson(items []map[string]types.AttributeValue, keySchema []string) ([
for i, m := range items {
id, err := dynamoKeyToIdBson(m, keySchema)
if err != nil {
return nil, err
return nil, fmt.Errorf("err in key to bson: %w", err)
}
b, err := toBson(&types.AttributeValueMemberM{Value: m})
if err != nil {
return nil, err
return nil, fmt.Errorf("err in to bson: %w", err)
}

// TODO: We currently clobber any existing _id
Expand Down Expand Up @@ -262,6 +267,8 @@ func streamTypeToDynamoType(st streamtypes.AttributeValue) (types.AttributeValue
return &types.AttributeValueMemberS{Value: tv.Value}, nil
case *streamtypes.AttributeValueMemberSS:
return &types.AttributeValueMemberSS{Value: tv.Value}, nil
case *streamtypes.AttributeValueMemberNULL:
return &types.AttributeValueMemberNULL{Value: tv.Value}, nil
default:
return nil, fmt.Errorf("unknown attribute %T", st)
}
Expand All @@ -270,21 +277,21 @@ func streamTypeToDynamoType(st streamtypes.AttributeValue) (types.AttributeValue
func dynamoWriteKeyValue(w io.Writer, av types.AttributeValue) error {
switch tv := av.(type) {
case *types.AttributeValueMemberB:
if err := binary.Write(w, binary.BigEndian, len(tv.Value)); err != nil {
if err := binary.Write(w, binary.BigEndian, int32(len(tv.Value))); err != nil {
return err
}
if _, err := w.Write(tv.Value); err != nil {
return err
}
case *types.AttributeValueMemberN:
if err := binary.Write(w, binary.BigEndian, len(tv.Value)); err != nil {
if err := binary.Write(w, binary.BigEndian, int32(len(tv.Value))); err != nil {
return err
}
if _, err := w.Write([]byte(tv.Value)); err != nil {
return err
}
case *types.AttributeValueMemberS:
if err := binary.Write(w, binary.BigEndian, len(tv.Value)); err != nil {
if err := binary.Write(w, binary.BigEndian, int32(len(tv.Value))); err != nil {
return err
}
if _, err := w.Write([]byte(tv.Value)); err != nil {
Expand Down
12 changes: 6 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ toolchain go1.24.3
require (
connectrpc.com/connect v1.18.1
connectrpc.com/grpcreflect v1.3.0
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2 v1.41.1
github.com/aws/aws-sdk-go-v2/config v1.29.17
github.com/aws/aws-sdk-go-v2/credentials v1.17.70
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.54.0
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1
github.com/aws/aws-sdk-go-v2/service/s3vectors v1.6.1
Expand Down Expand Up @@ -48,13 +48,13 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
Expand Down
24 changes: 12 additions & 12 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:W
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
Expand All @@ -29,24 +29,24 @@ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3 h1:xQYRnbQ+
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3/go.mod h1:X7RC8FFkx0bjNJRBddd3xdoDaDmNLSxICFdIdJ7asqw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4 h1:Rv6o9v2AfdEIKoAa7pQpJ5ch9ji2HevFUvGY6ufawlI=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4/go.mod h1:mWB0GE1bqcVSvpW7OtFA0sKuHk52+IqtnsYU2jUfYAs=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.54.0 h1:SW3MUVGaqOv/h4spv3IubyGz9CpvE0gHWEJsZQNPFMs=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.54.0/go.mod h1:ctEsEHY2vFQc6i4KU07q4n68v7BAmTbujv2Y+z8+hQY=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6 h1:QHaS/SHXfyNycuu4GiWb+AfW5T3bput6X5E3Ai/Q31M=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6/go.mod h1:He/RikglWUczbkV+fkdpcV/3GdL/rTRNVy7VaUiezMo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 h1:x187MqiHwBGjMGAed8Y8K1VGuCtFvQvXb24r+bwmSdo=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17/go.mod h1:mC9qMbA6e1pwEq6X3zDGtZRXMG2YaElJkbJlMVHLs5I=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.17 h1:Nhx/OYX+ukejm9t/MkWI8sucnsiroNYNGb5ddI9ungQ=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.17/go.mod h1:AjmK8JWnlAevq1b1NBtv5oQVG4iqnYXUufdgol+q9wg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
Expand Down
29 changes: 26 additions & 3 deletions internal/app/options/connectorflags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,15 +344,38 @@ func GetRegisteredConnectors() []RegisteredConnector {
IsConnector: func(s string) bool {
return strings.EqualFold(s, "dynamodb") || strings.EqualFold(s, "dynamodb://localstack")
},
Create: CreateHelper("DynamoDB", "dynamodb OR dynamodb://localstack", nil, func(_ *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) {
Create: CreateHelper("DynamoDB", "dynamodb OR dynamodb://localstack", []cli.Flag{
&cli.IntFlag{
Name: "doc-partition",
Usage: "Target number of documents per partition",
Value: 50000,
},
&cli.IntFlag{
Name: "plan-parallelism",
Usage: "Parallelism during planning",
Value: 4,
},
&cli.StringFlag{
Name: "id",
Usage: "A fixed id for the connector",
},
}, func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) {
if strings.EqualFold(args[0], "dynamodb://localstack") {
_, connString, ok := strings.Cut(args[0], "://")
if !ok {
return nil, fmt.Errorf("invalid connection string %v", args[0])
}
return dynamodb.NewConn(connString), nil
return dynamodb.NewConn(connString,
dynamodb.WithDocsPerSegment(c.Int("doc-partition")),
dynamodb.WithPlanParallelism(c.Int("plan-parallelism")),
dynamodb.WithID(c.String("id")),
), nil
} else {
return dynamodb.NewConn(""), nil
return dynamodb.NewConn("",
dynamodb.WithDocsPerSegment(c.Int("doc-partition")),
dynamodb.WithPlanParallelism(c.Int("plan-parallelism")),
dynamodb.WithID(c.String("id")),
), nil
}
}),
},
Expand Down