diff --git a/src/cli/commands/datastore_setup.ts b/src/cli/commands/datastore_setup.ts index db381b8b..4bdab03a 100644 --- a/src/cli/commands/datastore_setup.ts +++ b/src/cli/commands/datastore_setup.ts @@ -88,6 +88,14 @@ const datastoreSetupS3Command = new Command() .option("--bucket ", "S3 bucket name", { required: true }) .option("--prefix ", "Key prefix within the bucket") .option("--region ", "AWS region") + .option( + "--endpoint ", + "Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com)", + ) + .option( + "--force-path-style", + "Use path-style addressing (bucket in path, not subdomain)", + ) .option("--skip-migration", "Skip pushing existing data to S3") .action(async function (options: AnyOptions) { const cliCtx = createContext(options as GlobalOptions, [ @@ -115,6 +123,8 @@ const datastoreSetupS3Command = new Command() bucket: options.bucket, prefix: options.prefix, region: options.region, + endpoint: options.endpoint, + forcePathStyle: options.forcePathStyle ? true : undefined, repoDir, repoId: marker?.repoId, skipMigration: !!options.skipMigration, diff --git a/src/cli/repo_context.ts b/src/cli/repo_context.ts index 9f953387..dd754f2a 100644 --- a/src/cli/repo_context.ts +++ b/src/cli/repo_context.ts @@ -334,6 +334,8 @@ export function requireInitializedRepo( bucket: datastoreConfig.bucket, prefix: datastoreConfig.prefix, region: datastoreConfig.region, + endpoint: datastoreConfig.endpoint, + forcePathStyle: datastoreConfig.forcePathStyle, }); const lock = new S3Lock(s3); diff --git a/src/cli/resolve_datastore.ts b/src/cli/resolve_datastore.ts index 3780ef80..5459eac4 100644 --- a/src/cli/resolve_datastore.ts +++ b/src/cli/resolve_datastore.ts @@ -192,6 +192,8 @@ export function resolveDatastoreConfig( bucket: ds.bucket, prefix: ds.prefix, region: ds.region, + endpoint: ds.endpoint, + forcePathStyle: ds.forcePathStyle, cachePath, directories: ds.directories, exclude: ds.exclude, diff --git a/src/cli/resolve_datastore_test.ts b/src/cli/resolve_datastore_test.ts index eae517c1..38b7f3a3 100644 --- a/src/cli/resolve_datastore_test.ts +++ b/src/cli/resolve_datastore_test.ts @@ -190,6 +190,53 @@ Deno.test("resolveDatastoreConfig - marker config used when no env/cli", () => { } }); +// ============================================================================ +// S3 endpoint / forcePathStyle tests +// ============================================================================ + +Deno.test("resolveDatastoreConfig - S3 marker with endpoint and forcePathStyle", () => { + const marker: RepoMarkerData = { + swampVersion: "0.1.0", + initializedAt: "2024-01-01", + repoId: "test-repo", + datastore: { + type: "s3", + bucket: "my-space", + region: "us-east-1", + endpoint: "https://nyc3.digitaloceanspaces.com", + forcePathStyle: false, + }, + }; + const config = resolveDatastoreConfig(marker, undefined, "/repo"); + assertEquals(config.type, "s3"); + if (!isCustomDatastoreConfig(config) && config.type === "s3") { + assertEquals(config.bucket, "my-space"); + assertEquals(config.region, "us-east-1"); + assertEquals(config.endpoint, "https://nyc3.digitaloceanspaces.com"); + assertEquals(config.forcePathStyle, false); + } +}); + +Deno.test("resolveDatastoreConfig - S3 marker without endpoint defaults to undefined", () => { + const marker: RepoMarkerData = { + swampVersion: "0.1.0", + initializedAt: "2024-01-01", + repoId: "test-repo", + datastore: { + type: "s3", + bucket: "my-bucket", + region: "us-west-2", + }, + }; + const config = resolveDatastoreConfig(marker, undefined, "/repo"); + assertEquals(config.type, "s3"); + if (!isCustomDatastoreConfig(config) && config.type === "s3") { + assertEquals(config.bucket, "my-bucket"); + assertEquals(config.endpoint, undefined); + assertEquals(config.forcePathStyle, undefined); + } +}); + // ============================================================================ // Custom datastore type tests // ============================================================================ diff --git a/src/domain/datastore/datastore_config.ts b/src/domain/datastore/datastore_config.ts index 30833d09..41ecf774 100644 --- a/src/domain/datastore/datastore_config.ts +++ b/src/domain/datastore/datastore_config.ts @@ -80,6 +80,10 @@ export interface S3DatastoreConfig { readonly prefix?: string; /** AWS region */ readonly region?: string; + /** Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com) */ + readonly endpoint?: string; + /** Use path-style addressing (bucket in path, not subdomain). Default: false. */ + readonly forcePathStyle?: boolean; /** Local cache directory path (defaults to ~/.swamp/repos/{repoId}/) */ readonly cachePath: string; /** Which subdirectories belong to the datastore (defaults to DEFAULT_DATASTORE_SUBDIRS) */ @@ -127,6 +131,8 @@ export interface DatastoreConfigData { bucket?: string; prefix?: string; region?: string; + endpoint?: string; + forcePathStyle?: boolean; config?: Record; directories?: string[]; exclude?: string[]; diff --git a/src/infrastructure/persistence/s3_client.ts b/src/infrastructure/persistence/s3_client.ts index 72125bc1..87dc1b2f 100644 --- a/src/infrastructure/persistence/s3_client.ts +++ b/src/infrastructure/persistence/s3_client.ts @@ -35,6 +35,10 @@ export interface S3ClientConfig { bucket: string; prefix?: string; region?: string; + /** Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com) */ + endpoint?: string; + /** Use path-style addressing (bucket in path, not subdomain). Default: false. */ + forcePathStyle?: boolean; } export interface S3ListResult { @@ -54,6 +58,10 @@ export class S3Client { constructor(config: S3ClientConfig) { this.client = new AwsS3Client({ region: config.region, + ...(config.endpoint ? { endpoint: config.endpoint } : {}), + ...(config.forcePathStyle != null + ? { forcePathStyle: config.forcePathStyle } + : {}), }); this.bucket = config.bucket; this.prefix = config.prefix ?? ""; diff --git a/src/infrastructure/persistence/s3_datastore_verifier.ts b/src/infrastructure/persistence/s3_datastore_verifier.ts index d16ec298..c09ef3ae 100644 --- a/src/infrastructure/persistence/s3_datastore_verifier.ts +++ b/src/infrastructure/persistence/s3_datastore_verifier.ts @@ -33,9 +33,21 @@ export class S3DatastoreVerifier implements DatastoreVerifier { private readonly s3: S3Client; private readonly bucket: string; - constructor(bucket: string, prefix?: string, region?: string) { + constructor( + bucket: string, + prefix?: string, + region?: string, + endpoint?: string, + forcePathStyle?: boolean, + ) { this.bucket = bucket; - this.s3 = new S3Client({ bucket, prefix, region }); + this.s3 = new S3Client({ + bucket, + prefix, + region, + endpoint, + forcePathStyle, + }); } async verify(): Promise { diff --git a/src/libswamp/datastores/setup.ts b/src/libswamp/datastores/setup.ts index 0699f773..056c7b26 100644 --- a/src/libswamp/datastores/setup.ts +++ b/src/libswamp/datastores/setup.ts @@ -71,6 +71,8 @@ export interface DatastoreSetupS3Input { bucket: string; prefix?: string; region?: string; + endpoint?: string; + forcePathStyle?: boolean; repoDir: string; repoId?: string; skipMigration: boolean; @@ -86,11 +88,15 @@ export interface DatastoreSetupDeps { bucket: string, prefix?: string, region?: string, + endpoint?: string, + forcePathStyle?: boolean, ) => Promise<{ healthy: boolean; message: string }>; checkS3DatastoreExists: ( bucket: string, prefix?: string, region?: string, + endpoint?: string, + forcePathStyle?: boolean, ) => Promise; ensureDir: (path: string) => Promise; getDatastoreDirectories: (config: { @@ -122,6 +128,8 @@ export interface DatastoreSetupDeps { prefix: string | undefined, region: string | undefined, cachePath: string, + endpoint?: string, + forcePathStyle?: boolean, ) => Promise; getSwampDataDir: () => string; getCachePath: (repoId: string) => string; @@ -271,6 +279,8 @@ export async function* datastoreSetupS3( input.bucket, input.prefix, input.region, + input.endpoint, + input.forcePathStyle, ); if (!health.healthy) { yield { @@ -288,6 +298,8 @@ export async function* datastoreSetupS3( input.bucket, input.prefix, input.region, + input.endpoint, + input.forcePathStyle, ); if (exists) { const prefixStr = input.prefix ?? ""; @@ -315,6 +327,8 @@ export async function* datastoreSetupS3( bucket: input.bucket, prefix: input.prefix, region: input.region, + endpoint: input.endpoint, + forcePathStyle: input.forcePathStyle, }); // Migrate existing data to S3 @@ -345,6 +359,8 @@ export async function* datastoreSetupS3( input.prefix, input.region, cachePath, + input.endpoint, + input.forcePathStyle, ); ctx.logger.debug`Pushed ${filesPushed} file(s) to S3`; } catch (error) { @@ -443,16 +459,32 @@ export function createDatastoreSetupDeps( bucket: string, prefix?: string, region?: string, + endpoint?: string, + forcePathStyle?: boolean, ) => { - const verifier = new S3DatastoreVerifier(bucket, prefix, region); + const verifier = new S3DatastoreVerifier( + bucket, + prefix, + region, + endpoint, + forcePathStyle, + ); return await verifier.verify(); }, checkS3DatastoreExists: async ( bucket: string, prefix?: string, region?: string, + endpoint?: string, + forcePathStyle?: boolean, ) => { - const s3 = new S3Client({ bucket, prefix, region }); + const s3 = new S3Client({ + bucket, + prefix, + region, + endpoint, + forcePathStyle, + }); try { await s3.getObject(".datastore-index.json"); return true; @@ -502,8 +534,16 @@ export function createDatastoreSetupDeps( prefix: string | undefined, region: string | undefined, cachePath: string, + endpoint?: string, + forcePathStyle?: boolean, ) => { - const s3 = new S3Client({ bucket, prefix, region }); + const s3 = new S3Client({ + bucket, + prefix, + region, + endpoint, + forcePathStyle, + }); const syncService = new S3CacheSyncService(s3, cachePath); return await syncService.pushAll(); }, diff --git a/src/libswamp/datastores/status.ts b/src/libswamp/datastores/status.ts index 9df8b664..6ad95131 100644 --- a/src/libswamp/datastores/status.ts +++ b/src/libswamp/datastores/status.ts @@ -39,6 +39,7 @@ export interface DatastoreStatusData { bucket?: string; prefix?: string; region?: string; + endpoint?: string; healthy: boolean; message: string; latencyMs: number; @@ -86,6 +87,8 @@ export function createDatastoreStatusDeps( config.bucket, config.prefix, config.region, + config.endpoint, + config.forcePathStyle, ); return await verifier.verify(); } @@ -125,6 +128,9 @@ export async function* datastoreStatus( region: !isCustomDatastoreConfig(config) && config.type === "s3" ? config.region : undefined, + endpoint: !isCustomDatastoreConfig(config) && config.type === "s3" + ? config.endpoint + : undefined, healthy, message, latencyMs, diff --git a/src/libswamp/datastores/status_test.ts b/src/libswamp/datastores/status_test.ts index d313b921..77d7fa8a 100644 --- a/src/libswamp/datastores/status_test.ts +++ b/src/libswamp/datastores/status_test.ts @@ -55,6 +55,7 @@ Deno.test("datastoreStatus: healthy filesystem datastore", async () => { bucket: undefined, prefix: undefined, region: undefined, + endpoint: undefined, healthy: true, message: "OK", latencyMs: 5, @@ -89,6 +90,7 @@ Deno.test("datastoreStatus: healthy S3 datastore", async () => { bucket: "my-bucket", prefix: "swamp/", region: "us-east-1", + endpoint: undefined, healthy: true, message: "OK", latencyMs: 120, @@ -119,6 +121,7 @@ Deno.test("datastoreStatus: unhealthy datastore", async () => { bucket: undefined, prefix: undefined, region: undefined, + endpoint: undefined, healthy: false, message: "Connection refused", latencyMs: 0, @@ -149,6 +152,7 @@ Deno.test("datastoreStatus: includes exclude patterns", async () => { bucket: undefined, prefix: undefined, region: undefined, + endpoint: undefined, healthy: true, message: "OK", latencyMs: 5, diff --git a/src/libswamp/datastores/sync.ts b/src/libswamp/datastores/sync.ts index 4bd1f5c9..5ed1ed4a 100644 --- a/src/libswamp/datastores/sync.ts +++ b/src/libswamp/datastores/sync.ts @@ -122,6 +122,8 @@ export function createDatastoreSyncDeps( bucket: config.bucket, prefix: config.prefix, region: config.region, + endpoint: config.endpoint, + forcePathStyle: config.forcePathStyle, }); const syncService = new S3CacheSyncService(s3, config.cachePath); diff --git a/src/presentation/renderers/datastore_status.ts b/src/presentation/renderers/datastore_status.ts index 01dbfc12..15353488 100644 --- a/src/presentation/renderers/datastore_status.ts +++ b/src/presentation/renderers/datastore_status.ts @@ -51,6 +51,9 @@ class LogDatastoreStatusRenderer implements Renderer { if (data.region) { lines.push(` Region: ${data.region}`); } + if (data.endpoint) { + lines.push(` Endpoint: ${data.endpoint}`); + } lines.push( ` Health: ${healthIcon} ${healthText} (${ Math.round(data.latencyMs)