diff --git a/.changeset/object-versions.md b/.changeset/object-versions.md new file mode 100644 index 0000000..0444734 --- /dev/null +++ b/.changeset/object-versions.md @@ -0,0 +1,10 @@ +--- +'@tigrisdata/storage': minor +--- + +Add support for S3 object-version operations on snapshot-enabled buckets: + +- New `listVersions({ prefix, delimiter, limit, keyMarker, versionIdMarker })` returns the object versions and delete markers for a bucket prefix, plus `nextKeyMarker` / `nextVersionIdMarker` for pagination. +- `head`, `get`, and `remove` now accept a `versionId` option. On `head` / `get` it selects the specified version; on `remove` it permanently deletes that version (without one, `remove` creates a delete marker on a versioned bucket, matching S3 semantics). + +Versioning is enabled implicitly by snapshots — pass `enableSnapshot: true` to `createBucket`. diff --git a/packages/storage/src/lib/object/get.ts b/packages/storage/src/lib/object/get.ts index 7cd312f..0f9d9fb 100644 --- a/packages/storage/src/lib/object/get.ts +++ b/packages/storage/src/lib/object/get.ts @@ -11,6 +11,7 @@ export type GetOptions = { contentType?: string; encoding?: string; snapshotVersion?: string; + versionId?: string; }; export type GetResponse = string | File | ReadableStream; @@ -44,6 +45,7 @@ export async function get( const get = new GetObjectCommand({ Bucket: options?.config?.bucket ?? config.bucket, Key: path, + VersionId: options?.versionId, ResponseContentType: options?.contentType ?? undefined, ResponseContentDisposition: options?.contentDisposition ? options.contentDisposition === 'attachment' diff --git a/packages/storage/src/lib/object/head.ts b/packages/storage/src/lib/object/head.ts index a32f832..dbe9d3a 100644 --- a/packages/storage/src/lib/object/head.ts +++ b/packages/storage/src/lib/object/head.ts @@ -8,6 +8,7 @@ import type { TigrisStorageConfig, TigrisStorageResponse } from '../types'; export type HeadOptions = { snapshotVersion?: string; + versionId?: string; config?: TigrisStorageConfig; }; @@ -33,6 +34,7 @@ export async function head( const head = new HeadObjectCommand({ Bucket: options?.config?.bucket ?? config.bucket, Key: path, + VersionId: options?.versionId, }); if (options?.snapshotVersion) { diff --git a/packages/storage/src/lib/object/list-versions.ts b/packages/storage/src/lib/object/list-versions.ts new file mode 100644 index 0000000..65ae41b --- /dev/null +++ b/packages/storage/src/lib/object/list-versions.ts @@ -0,0 +1,104 @@ +import { ListObjectVersionsCommand } from '@aws-sdk/client-s3'; +import { handleError } from '@shared/utils'; +import { config, missingConfigError } from '../config'; +import { createTigrisClient } from '../tigris-client'; +import type { TigrisStorageConfig, TigrisStorageResponse } from '../types'; + +export type ListVersionsOptions = { + delimiter?: string; + prefix?: string; + limit?: number; + keyMarker?: string; + /** + * Pagination position within a key's versions. S3 ignores this when + * `keyMarker` is not also set, so passing it alone returns an error. + */ + versionIdMarker?: string; + config?: TigrisStorageConfig; +}; + +export type ObjectVersion = { + name: string; + versionId: string | undefined; + isLatest: boolean; + size: number; + lastModified: Date; +}; + +export type DeleteMarker = { + name: string; + versionId: string | undefined; + isLatest: boolean; + lastModified: Date; +}; + +export type ListVersionsResponse = { + versions: ObjectVersion[]; + deleteMarkers: DeleteMarker[]; + commonPrefixes: string[]; + nextKeyMarker: string | undefined; + nextVersionIdMarker: string | undefined; + hasMore: boolean; +}; + +export async function listVersions( + options?: ListVersionsOptions +): Promise> { + if (!options?.config?.bucket && !config.bucket) { + return missingConfigError('bucket'); + } + + if (options?.versionIdMarker && !options?.keyMarker) { + return { + error: new Error('versionIdMarker requires keyMarker to also be set'), + }; + } + + const { data: tigrisClient, error } = createTigrisClient(options?.config); + + if (error) { + return { error }; + } + + const command = new ListObjectVersionsCommand({ + Bucket: options?.config?.bucket ?? config.bucket, + Prefix: options?.prefix, + Delimiter: options?.delimiter, + MaxKeys: options?.limit, + KeyMarker: options?.keyMarker, + VersionIdMarker: options?.versionIdMarker, + }); + + try { + return tigrisClient + .send(command) + .then((res) => ({ + data: { + versions: + res.Versions?.map((v) => ({ + name: v.Key ?? '', + versionId: v.VersionId, + isLatest: v.IsLatest ?? false, + size: v.Size ?? 0, + lastModified: v.LastModified ?? new Date(), + })) ?? [], + deleteMarkers: + res.DeleteMarkers?.map((m) => ({ + name: m.Key ?? '', + versionId: m.VersionId, + isLatest: m.IsLatest ?? false, + lastModified: m.LastModified ?? new Date(), + })) ?? [], + commonPrefixes: + res.CommonPrefixes?.map((p) => p.Prefix ?? '').filter(Boolean) ?? + [], + nextKeyMarker: res.NextKeyMarker, + nextVersionIdMarker: res.NextVersionIdMarker, + hasMore: res.IsTruncated ?? false, + }, + })) + .catch(handleError); + } catch (error) { + return handleError(error as Error); + } +} diff --git a/packages/storage/src/lib/object/remove.ts b/packages/storage/src/lib/object/remove.ts index 5296904..4b9c613 100644 --- a/packages/storage/src/lib/object/remove.ts +++ b/packages/storage/src/lib/object/remove.ts @@ -6,6 +6,7 @@ import type { TigrisStorageConfig, TigrisStorageResponse } from '../types'; export type RemoveOptions = { config?: TigrisStorageConfig; + versionId?: string; }; export async function remove( @@ -20,6 +21,7 @@ export async function remove( const remove = new DeleteObjectCommand({ Bucket: options?.config?.bucket ?? config.bucket, Key: path, + VersionId: options?.versionId, }); try { diff --git a/packages/storage/src/server.ts b/packages/storage/src/server.ts index 15cbbb1..7347a9e 100644 --- a/packages/storage/src/server.ts +++ b/packages/storage/src/server.ts @@ -73,6 +73,13 @@ export { type ListResponse, list, } from './lib/object/list'; +export { + type DeleteMarker, + type ListVersionsOptions, + type ListVersionsResponse, + listVersions, + type ObjectVersion, +} from './lib/object/list-versions'; export { isMigrated, type MigrateOptions, migrate } from './lib/object/migrate'; export { type MoveOptions, type MoveResponse, move } from './lib/object/move'; export { diff --git a/packages/storage/src/test/object-versions.integration.test.ts b/packages/storage/src/test/object-versions.integration.test.ts new file mode 100644 index 0000000..1262ccf --- /dev/null +++ b/packages/storage/src/test/object-versions.integration.test.ts @@ -0,0 +1,164 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { createBucket } from '../lib/bucket/create'; +import { removeBucket } from '../lib/bucket/remove'; +import { config } from '../lib/config'; +import { get } from '../lib/object/get'; +import { head } from '../lib/object/head'; +import { listVersions } from '../lib/object/list-versions'; +import { put } from '../lib/object/put'; +import { remove } from '../lib/object/remove'; +import { shouldSkipIntegrationTests } from './setup'; + +const skipTests = shouldSkipIntegrationTests(); + +describe.skipIf(skipTests)('Object versioning Integration Tests', () => { + const bucket = `test-versions-${Date.now()}`.toLowerCase(); + const bucketConfig = { ...config, bucket }; + + beforeAll(async () => { + const result = await createBucket(bucket, { + enableSnapshot: true, + config, + }); + expect( + result.error, + `bucket create failed: ${result.error?.message}` + ).toBeUndefined(); + }); + + afterAll(async () => { + await removeBucket(bucket, { force: true, config }); + }); + + it('lists multiple versions of the same key', async () => { + const key = `nested/key-${Date.now()}.txt`; + + await put(key, 'v1', { config: bucketConfig }); + await put(key, 'v2', { config: bucketConfig }); + + const result = await listVersions({ + prefix: key, + config: bucketConfig, + }); + + expect(result.error).toBeUndefined(); + expect(result.data?.versions).toHaveLength(2); + expect(result.data?.deleteMarkers).toHaveLength(0); + + const versions = result.data?.versions ?? []; + expect(versions.filter((v) => v.isLatest)).toHaveLength(1); + for (const v of versions) { + expect(v.name).toBe(key); + expect(v.versionId).toBeTruthy(); + expect(v.lastModified).toBeInstanceOf(Date); + } + }); + + it('reads a prior version with get(versionId) and head(versionId)', async () => { + const key = `read-prev-${Date.now()}.txt`; + + await put(key, 'old', { config: bucketConfig }); + await put(key, 'new', { config: bucketConfig }); + + const { data: list } = await listVersions({ + prefix: key, + config: bucketConfig, + }); + const older = list?.versions.find((v) => !v.isLatest); + expect(older).toBeDefined(); + + const getRes = await get(key, 'string', { + versionId: older?.versionId, + config: bucketConfig, + }); + expect(getRes.error).toBeUndefined(); + expect(getRes.data).toBe('old'); + + const latest = await get(key, 'string', { config: bucketConfig }); + expect(latest.data).toBe('new'); + + const headRes = await head(key, { + versionId: older?.versionId, + config: bucketConfig, + }); + expect(headRes.error).toBeUndefined(); + expect(headRes.data?.path).toBe(key); + }); + + it('remove() without versionId creates a delete marker', async () => { + const key = `del-marker-${Date.now()}.txt`; + await put(key, 'v1', { config: bucketConfig }); + + const removeRes = await remove(key, { config: bucketConfig }); + expect(removeRes.error).toBeUndefined(); + + const { data } = await listVersions({ + prefix: key, + config: bucketConfig, + }); + expect(data?.versions).toHaveLength(1); + expect(data?.deleteMarkers).toHaveLength(1); + expect(data?.deleteMarkers[0]?.isLatest).toBe(true); + }); + + it('remove() with versionId permanently deletes that version', async () => { + const key = `perm-del-${Date.now()}.txt`; + await put(key, 'v1', { config: bucketConfig }); + await put(key, 'v2', { config: bucketConfig }); + + const { data: before } = await listVersions({ + prefix: key, + config: bucketConfig, + }); + expect(before?.versions).toHaveLength(2); + + const older = before?.versions.find((v) => !v.isLatest); + expect(older?.versionId).toBeTruthy(); + + const removeRes = await remove(key, { + versionId: older?.versionId, + config: bucketConfig, + }); + expect(removeRes.error).toBeUndefined(); + + const { data: after } = await listVersions({ + prefix: key, + config: bucketConfig, + }); + expect(after?.versions).toHaveLength(1); + expect(after?.deleteMarkers).toHaveLength(0); + expect(after?.versions.some((v) => v.versionId === older?.versionId)).toBe( + false + ); + }); + + it('paginates with keyMarker / versionIdMarker', async () => { + const prefix = `page-${Date.now()}/`; + const keys = [`${prefix}a.txt`, `${prefix}b.txt`, `${prefix}c.txt`]; + for (const k of keys) { + await put(k, 'x', { config: bucketConfig }); + } + + const first = await listVersions({ + prefix, + limit: 2, + config: bucketConfig, + }); + expect(first.data?.hasMore).toBe(true); + expect(first.data?.versions).toHaveLength(2); + expect(first.data?.nextKeyMarker).toBeTruthy(); + + const second = await listVersions({ + prefix, + keyMarker: first.data?.nextKeyMarker, + versionIdMarker: first.data?.nextVersionIdMarker, + config: bucketConfig, + }); + expect(second.data?.hasMore).toBe(false); + expect(second.data?.versions.length).toBeGreaterThanOrEqual(1); + + const firstNames = first.data?.versions.map((v) => v.name) ?? []; + const secondNames = second.data?.versions.map((v) => v.name) ?? []; + expect(firstNames.some((n) => secondNames.includes(n))).toBe(false); + }); +});