Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/cli/commands/datastore_setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ const datastoreSetupS3Command = new Command()
.option("--bucket <bucket:string>", "S3 bucket name", { required: true })
.option("--prefix <prefix:string>", "Key prefix within the bucket")
.option("--region <region:string>", "AWS region")
.option(
"--endpoint <endpoint:string>",
"Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com)",
)
.option(
"--force-path-style",
"Use path-style addressing (bucket in path, not subdomain)",
)
.option("--skip-migration", "Skip pushing existing data to S3")
.action(async function (options: AnyOptions) {
const cliCtx = createContext(options as GlobalOptions, [
Expand Down Expand Up @@ -115,6 +123,8 @@ const datastoreSetupS3Command = new Command()
bucket: options.bucket,
prefix: options.prefix,
region: options.region,
endpoint: options.endpoint,
forcePathStyle: options.forcePathStyle ? true : undefined,
repoDir,
repoId: marker?.repoId,
skipMigration: !!options.skipMigration,
Expand Down
2 changes: 2 additions & 0 deletions src/cli/repo_context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,8 @@ export function requireInitializedRepo(
bucket: datastoreConfig.bucket,
prefix: datastoreConfig.prefix,
region: datastoreConfig.region,
endpoint: datastoreConfig.endpoint,
forcePathStyle: datastoreConfig.forcePathStyle,
});

const lock = new S3Lock(s3);
Expand Down
2 changes: 2 additions & 0 deletions src/cli/resolve_datastore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ export function resolveDatastoreConfig(
bucket: ds.bucket,
prefix: ds.prefix,
region: ds.region,
endpoint: ds.endpoint,
forcePathStyle: ds.forcePathStyle,
cachePath,
directories: ds.directories,
exclude: ds.exclude,
Expand Down
47 changes: 47 additions & 0 deletions src/cli/resolve_datastore_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,53 @@ Deno.test("resolveDatastoreConfig - marker config used when no env/cli", () => {
}
});

// ============================================================================
// S3 endpoint / forcePathStyle tests
// ============================================================================

Deno.test("resolveDatastoreConfig - S3 marker with endpoint and forcePathStyle", () => {
const marker: RepoMarkerData = {
swampVersion: "0.1.0",
initializedAt: "2024-01-01",
repoId: "test-repo",
datastore: {
type: "s3",
bucket: "my-space",
region: "us-east-1",
endpoint: "https://nyc3.digitaloceanspaces.com",
forcePathStyle: false,
},
};
const config = resolveDatastoreConfig(marker, undefined, "/repo");
assertEquals(config.type, "s3");
if (!isCustomDatastoreConfig(config) && config.type === "s3") {
assertEquals(config.bucket, "my-space");
assertEquals(config.region, "us-east-1");
assertEquals(config.endpoint, "https://nyc3.digitaloceanspaces.com");
assertEquals(config.forcePathStyle, false);
}
});

Deno.test("resolveDatastoreConfig - S3 marker without endpoint defaults to undefined", () => {
const marker: RepoMarkerData = {
swampVersion: "0.1.0",
initializedAt: "2024-01-01",
repoId: "test-repo",
datastore: {
type: "s3",
bucket: "my-bucket",
region: "us-west-2",
},
};
const config = resolveDatastoreConfig(marker, undefined, "/repo");
assertEquals(config.type, "s3");
if (!isCustomDatastoreConfig(config) && config.type === "s3") {
assertEquals(config.bucket, "my-bucket");
assertEquals(config.endpoint, undefined);
assertEquals(config.forcePathStyle, undefined);
}
});

// ============================================================================
// Custom datastore type tests
// ============================================================================
Expand Down
6 changes: 6 additions & 0 deletions src/domain/datastore/datastore_config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ export interface S3DatastoreConfig {
readonly prefix?: string;
/** AWS region */
readonly region?: string;
/** Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com) */
readonly endpoint?: string;
/** Use path-style addressing (bucket in path, not subdomain). Default: false. */
readonly forcePathStyle?: boolean;
/** Local cache directory path (defaults to ~/.swamp/repos/{repoId}/) */
readonly cachePath: string;
/** Which subdirectories belong to the datastore (defaults to DEFAULT_DATASTORE_SUBDIRS) */
Expand Down Expand Up @@ -127,6 +131,8 @@ export interface DatastoreConfigData {
bucket?: string;
prefix?: string;
region?: string;
endpoint?: string;
forcePathStyle?: boolean;
config?: Record<string, unknown>;
directories?: string[];
exclude?: string[];
Expand Down
8 changes: 8 additions & 0 deletions src/infrastructure/persistence/s3_client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ export interface S3ClientConfig {
bucket: string;
prefix?: string;
region?: string;
/** Custom S3-compatible endpoint URL (e.g., https://nyc3.digitaloceanspaces.com) */
endpoint?: string;
/** Use path-style addressing (bucket in path, not subdomain). Default: false. */
forcePathStyle?: boolean;
}

export interface S3ListResult {
Expand All @@ -54,6 +58,10 @@ export class S3Client {
constructor(config: S3ClientConfig) {
this.client = new AwsS3Client({
region: config.region,
...(config.endpoint ? { endpoint: config.endpoint } : {}),
...(config.forcePathStyle != null
? { forcePathStyle: config.forcePathStyle }
: {}),
});
this.bucket = config.bucket;
this.prefix = config.prefix ?? "";
Expand Down
16 changes: 14 additions & 2 deletions src/infrastructure/persistence/s3_datastore_verifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,21 @@ export class S3DatastoreVerifier implements DatastoreVerifier {
private readonly s3: S3Client;
private readonly bucket: string;

constructor(bucket: string, prefix?: string, region?: string) {
constructor(
bucket: string,
prefix?: string,
region?: string,
endpoint?: string,
forcePathStyle?: boolean,
) {
this.bucket = bucket;
this.s3 = new S3Client({ bucket, prefix, region });
this.s3 = new S3Client({
bucket,
prefix,
region,
endpoint,
forcePathStyle,
});
}

async verify(): Promise<DatastoreHealthResult> {
Expand Down
46 changes: 43 additions & 3 deletions src/libswamp/datastores/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ export interface DatastoreSetupS3Input {
bucket: string;
prefix?: string;
region?: string;
endpoint?: string;
forcePathStyle?: boolean;
repoDir: string;
repoId?: string;
skipMigration: boolean;
Expand All @@ -86,11 +88,15 @@ export interface DatastoreSetupDeps {
bucket: string,
prefix?: string,
region?: string,
endpoint?: string,
forcePathStyle?: boolean,
) => Promise<{ healthy: boolean; message: string }>;
checkS3DatastoreExists: (
bucket: string,
prefix?: string,
region?: string,
endpoint?: string,
forcePathStyle?: boolean,
) => Promise<boolean>;
ensureDir: (path: string) => Promise<void>;
getDatastoreDirectories: (config: {
Expand Down Expand Up @@ -122,6 +128,8 @@ export interface DatastoreSetupDeps {
prefix: string | undefined,
region: string | undefined,
cachePath: string,
endpoint?: string,
forcePathStyle?: boolean,
) => Promise<number>;
getSwampDataDir: () => string;
getCachePath: (repoId: string) => string;
Expand Down Expand Up @@ -271,6 +279,8 @@ export async function* datastoreSetupS3(
input.bucket,
input.prefix,
input.region,
input.endpoint,
input.forcePathStyle,
);
if (!health.healthy) {
yield {
Expand All @@ -288,6 +298,8 @@ export async function* datastoreSetupS3(
input.bucket,
input.prefix,
input.region,
input.endpoint,
input.forcePathStyle,
);
if (exists) {
const prefixStr = input.prefix ?? "";
Expand Down Expand Up @@ -315,6 +327,8 @@ export async function* datastoreSetupS3(
bucket: input.bucket,
prefix: input.prefix,
region: input.region,
endpoint: input.endpoint,
forcePathStyle: input.forcePathStyle,
});

// Migrate existing data to S3
Expand Down Expand Up @@ -345,6 +359,8 @@ export async function* datastoreSetupS3(
input.prefix,
input.region,
cachePath,
input.endpoint,
input.forcePathStyle,
);
ctx.logger.debug`Pushed ${filesPushed} file(s) to S3`;
} catch (error) {
Expand Down Expand Up @@ -443,16 +459,32 @@ export function createDatastoreSetupDeps(
bucket: string,
prefix?: string,
region?: string,
endpoint?: string,
forcePathStyle?: boolean,
) => {
const verifier = new S3DatastoreVerifier(bucket, prefix, region);
const verifier = new S3DatastoreVerifier(
bucket,
prefix,
region,
endpoint,
forcePathStyle,
);
return await verifier.verify();
},
checkS3DatastoreExists: async (
bucket: string,
prefix?: string,
region?: string,
endpoint?: string,
forcePathStyle?: boolean,
) => {
const s3 = new S3Client({ bucket, prefix, region });
const s3 = new S3Client({
bucket,
prefix,
region,
endpoint,
forcePathStyle,
});
try {
await s3.getObject(".datastore-index.json");
return true;
Expand Down Expand Up @@ -502,8 +534,16 @@ export function createDatastoreSetupDeps(
prefix: string | undefined,
region: string | undefined,
cachePath: string,
endpoint?: string,
forcePathStyle?: boolean,
) => {
const s3 = new S3Client({ bucket, prefix, region });
const s3 = new S3Client({
bucket,
prefix,
region,
endpoint,
forcePathStyle,
});
const syncService = new S3CacheSyncService(s3, cachePath);
return await syncService.pushAll();
},
Expand Down
6 changes: 6 additions & 0 deletions src/libswamp/datastores/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export interface DatastoreStatusData {
bucket?: string;
prefix?: string;
region?: string;
endpoint?: string;
healthy: boolean;
message: string;
latencyMs: number;
Expand Down Expand Up @@ -86,6 +87,8 @@ export function createDatastoreStatusDeps(
config.bucket,
config.prefix,
config.region,
config.endpoint,
config.forcePathStyle,
);
return await verifier.verify();
}
Expand Down Expand Up @@ -125,6 +128,9 @@ export async function* datastoreStatus(
region: !isCustomDatastoreConfig(config) && config.type === "s3"
? config.region
: undefined,
endpoint: !isCustomDatastoreConfig(config) && config.type === "s3"
? config.endpoint
: undefined,
healthy,
message,
latencyMs,
Expand Down
4 changes: 4 additions & 0 deletions src/libswamp/datastores/status_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ Deno.test("datastoreStatus: healthy filesystem datastore", async () => {
bucket: undefined,
prefix: undefined,
region: undefined,
endpoint: undefined,
healthy: true,
message: "OK",
latencyMs: 5,
Expand Down Expand Up @@ -89,6 +90,7 @@ Deno.test("datastoreStatus: healthy S3 datastore", async () => {
bucket: "my-bucket",
prefix: "swamp/",
region: "us-east-1",
endpoint: undefined,
healthy: true,
message: "OK",
latencyMs: 120,
Expand Down Expand Up @@ -119,6 +121,7 @@ Deno.test("datastoreStatus: unhealthy datastore", async () => {
bucket: undefined,
prefix: undefined,
region: undefined,
endpoint: undefined,
healthy: false,
message: "Connection refused",
latencyMs: 0,
Expand Down Expand Up @@ -149,6 +152,7 @@ Deno.test("datastoreStatus: includes exclude patterns", async () => {
bucket: undefined,
prefix: undefined,
region: undefined,
endpoint: undefined,
healthy: true,
message: "OK",
latencyMs: 5,
Expand Down
2 changes: 2 additions & 0 deletions src/libswamp/datastores/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ export function createDatastoreSyncDeps(
bucket: config.bucket,
prefix: config.prefix,
region: config.region,
endpoint: config.endpoint,
forcePathStyle: config.forcePathStyle,
});
const syncService = new S3CacheSyncService(s3, config.cachePath);

Expand Down
3 changes: 3 additions & 0 deletions src/presentation/renderers/datastore_status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ class LogDatastoreStatusRenderer implements Renderer<DatastoreStatusEvent> {
if (data.region) {
lines.push(` Region: ${data.region}`);
}
if (data.endpoint) {
lines.push(` Endpoint: ${data.endpoint}`);
}
lines.push(
` Health: ${healthIcon} ${healthText} (${
Math.round(data.latencyMs)
Expand Down
Loading