From 3f2554edccb12aee16eb7952a5928d1e7afac1cd Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 5 May 2026 10:52:36 +0000 Subject: [PATCH 1/2] feat: add allow_custom_keys support, requestBulkUploadUrls mutation, auto-path derivation - Add optional 'key' input to requestUploadUrl for custom S3 keys - Custom keys require bucket.allow_custom_keys=true - Re-uploading to existing custom key auto-versions (previous_version_id) - Dedup by content_hash in both hash-based and custom-key modes - Auto-derive ltree path from custom key directory when has_path_shares=true - Add requestBulkUploadUrls mutation with per-storage-module limits - Add max_bulk_files and max_bulk_total_size to StorageModuleConfig - Add has_path_shares to StorageModuleConfig for path column awareness - Validate custom keys: no traversal, no leading slash, alphanumeric start --- .../src/plugin.ts | 513 ++++++++++++++---- .../src/storage-module-cache.ts | 20 +- .../src/types.ts | 19 + 3 files changed, 430 insertions(+), 122 deletions(-) diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 7497743cd..030095498 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -29,7 +29,9 @@ const log = new Logger('graphile-presigned-url:plugin'); const MAX_CONTENT_HASH_LENGTH = 128; const MAX_CONTENT_TYPE_LENGTH = 255; const MAX_BUCKET_KEY_LENGTH = 255; +const MAX_CUSTOM_KEY_LENGTH = 1024; const SHA256_HEX_REGEX = /^[a-f0-9]{64}$/; +const CUSTOM_KEY_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_.\-\/]*$/; // --- Helpers --- @@ -41,13 +43,48 @@ function isValidSha256(hash: string): boolean { } /** - * Build the S3 key from content hash and content type extension. + * Build the S3 key from content hash. * Format: {contentHash} (flat namespace, content-addressed) */ function buildS3Key(contentHash: string): string { return contentHash; } +/** + * Validate a custom S3 key. + * Must be 1-1024 chars, no path traversal, no leading slash, no null bytes. + */ +function validateCustomKey(key: string): string | null { + if (key.length === 0 || key.length > MAX_CUSTOM_KEY_LENGTH) { + return 'INVALID_KEY_LENGTH: must be 1-1024 characters'; + } + if (key.includes('..')) { + return 'INVALID_KEY: path traversal (..) not allowed'; + } + if (key.startsWith('/')) { + return 'INVALID_KEY: leading slash not allowed'; + } + if (key.includes('\0')) { + return 'INVALID_KEY: null bytes not allowed'; + } + if (!CUSTOM_KEY_REGEX.test(key)) { + return 'INVALID_KEY: must start with alphanumeric and contain only alphanumeric, dots, hyphens, underscores, and slashes'; + } + return null; +} + +/** + * Derive an ltree path from a custom S3 key's directory portion. + * e.g., "reports/2024/Q1/revenue.pdf" → "reports.2024.Q1" + * Returns null if the key has no directory component. + */ +function derivePathFromKey(key: string): string | null { + const lastSlash = key.lastIndexOf('/'); + if (lastSlash <= 0) return null; + const dir = key.substring(0, lastSlash); + return dir.replace(/\//g, '.'); +} + /** * Resolve the database_id from the JWT context. * The server middleware sets jwt.claims.database_id, which is accessible @@ -158,6 +195,14 @@ export function createPresignedUrlPlugin( size: Int! """Original filename (optional, for display and Content-Disposition)""" filename: String + """ + Custom S3 key (e.g., "reports/2024/Q1.pdf"). + Only allowed when the bucket has allow_custom_keys=true. + When omitted, key defaults to contentHash (content-addressed dedup). + When provided, the file is stored at this key. + Re-uploading to an existing key auto-creates a new version. + """ + key: String } type RequestUploadUrlPayload { @@ -171,6 +216,52 @@ export function createPresignedUrlPlugin( deduplicated: Boolean! """Presigned URL expiry time (null if deduplicated)""" expiresAt: Datetime + """ID of the previous version (set when re-uploading to an existing custom key)""" + previousVersionId: UUID + } + + input BulkUploadFileInput { + """SHA-256 content hash computed by the client (hex-encoded, 64 chars)""" + contentHash: String! + """MIME type of the file (e.g., "image/png")""" + contentType: String! + """File size in bytes""" + size: Int! + """Original filename (optional, for display and Content-Disposition)""" + filename: String + """Custom S3 key (only when bucket has allow_custom_keys=true)""" + key: String + } + + input RequestBulkUploadUrlsInput { + """Bucket key (e.g., "public", "private")""" + bucketKey: String! + """Owner entity ID for entity-scoped uploads""" + ownerId: UUID + """Array of files to upload""" + files: [BulkUploadFileInput!]! + } + + type BulkUploadFilePayload { + """Presigned PUT URL (null if file was deduplicated)""" + uploadUrl: String + """The file ID""" + fileId: UUID! + """The S3 object key""" + key: String! + """Whether this file was deduplicated""" + deduplicated: Boolean! + """Presigned URL expiry time (null if deduplicated)""" + expiresAt: Datetime + """ID of the previous version (set when re-uploading to an existing custom key)""" + previousVersionId: UUID + """Index of this file in the input array (for client correlation)""" + index: Int! + } + + type RequestBulkUploadUrlsPayload { + """Array of results, one per input file""" + files: [BulkUploadFilePayload!]! } extend type Mutation { @@ -183,6 +274,15 @@ export function createPresignedUrlPlugin( requestUploadUrl( input: RequestUploadUrlInput! ): RequestUploadUrlPayload + + """ + Request presigned URLs for uploading multiple files in a single batch. + Subject to per-storage-module limits (max_bulk_files, max_bulk_total_size). + Each file is processed independently — some may dedup while others get fresh URLs. + """ + requestBulkUploadUrls( + input: RequestBulkUploadUrlsInput! + ): RequestBulkUploadUrlsPayload } `, plans: { @@ -198,31 +298,37 @@ export function createPresignedUrlPlugin( }); return lambda($combined, async ({ input, withPgClient, pgSettings }: any) => { - // --- Input validation --- - const { bucketKey, ownerId, contentHash, contentType, size, filename } = input; + const result = await processUpload(options, input, withPgClient, pgSettings); + return result; + }); + }, + requestBulkUploadUrls(_$mutation: any, fieldArgs: any) { + const $input = fieldArgs.getRaw('input'); + const $withPgClient = (grafastContext() as any).get('withPgClient'); + const $pgSettings = (grafastContext() as any).get('pgSettings'); + const $combined = object({ + input: $input, + withPgClient: $withPgClient, + pgSettings: $pgSettings, + }); + + return lambda($combined, async ({ input, withPgClient, pgSettings }: any) => { + const { bucketKey, ownerId, files } = input; if (!bucketKey || typeof bucketKey !== 'string' || bucketKey.length > MAX_BUCKET_KEY_LENGTH) { throw new Error('INVALID_BUCKET_KEY'); } - if (!contentHash || typeof contentHash !== 'string' || contentHash.length > MAX_CONTENT_HASH_LENGTH) { - throw new Error('INVALID_CONTENT_HASH'); - } - if (!isValidSha256(contentHash)) { - throw new Error('INVALID_CONTENT_HASH_FORMAT: must be a 64-char lowercase hex SHA-256'); - } - if (!contentType || typeof contentType !== 'string' || contentType.length > MAX_CONTENT_TYPE_LENGTH) { - throw new Error('INVALID_CONTENT_TYPE'); + if (!Array.isArray(files) || files.length === 0) { + throw new Error('INVALID_FILES: must provide at least one file'); } return withPgClient(pgSettings, async (pgClient: any) => { return pgClient.withTransaction(async (txClient: any) => { - // --- Resolve storage module config (all limits come from here) --- const databaseId = await resolveDatabaseId(txClient); if (!databaseId) { throw new Error('DATABASE_NOT_FOUND'); } - // --- Resolve storage module (app-level or entity-scoped) --- const storageConfig = ownerId ? await getStorageModuleConfigForOwner(txClient, databaseId, ownerId) : await getStorageModuleConfig(txClient, databaseId); @@ -234,125 +340,40 @@ export function createPresignedUrlPlugin( ); } - // --- Validate size against storage module default (bucket override checked below) --- - if (typeof size !== 'number' || size <= 0 || size > storageConfig.defaultMaxFileSize) { - throw new Error(`INVALID_FILE_SIZE: must be between 1 and ${storageConfig.defaultMaxFileSize} bytes`); + // --- Validate bulk limits --- + if (files.length > storageConfig.maxBulkFiles) { + throw new Error(`BULK_LIMIT_EXCEEDED: max ${storageConfig.maxBulkFiles} files per batch`); } - if (filename !== undefined && filename !== null) { - if (typeof filename !== 'string' || filename.length > storageConfig.maxFilenameLength) { - throw new Error('INVALID_FILENAME'); - } + const totalSize = files.reduce((sum: number, f: any) => sum + (f.size || 0), 0); + if (totalSize > storageConfig.maxBulkTotalSize) { + throw new Error(`BULK_SIZE_EXCEEDED: total size ${totalSize} exceeds max ${storageConfig.maxBulkTotalSize} bytes`); } - // --- Look up the bucket (cached; first miss queries via RLS) --- const bucket = await getBucketConfig(txClient, storageConfig, databaseId, bucketKey, ownerId); if (!bucket) { throw new Error('BUCKET_NOT_FOUND'); } - // --- Validate content type against bucket's allowed_mime_types --- - if (bucket.allowed_mime_types && bucket.allowed_mime_types.length > 0) { - const allowed = bucket.allowed_mime_types as string[]; - const isAllowed = allowed.some((pattern: string) => { - if (pattern === '*/*') return true; - if (pattern.endsWith('/*')) { - const prefix = pattern.slice(0, -1); - return contentType.startsWith(prefix); - } - return contentType === pattern; - }); - if (!isAllowed) { - throw new Error(`CONTENT_TYPE_NOT_ALLOWED: ${contentType} not in bucket allowed types`); - } - } - - // --- Validate size against bucket's max_file_size --- - if (bucket.max_file_size && size > bucket.max_file_size) { - throw new Error(`FILE_TOO_LARGE: exceeds bucket max of ${bucket.max_file_size} bytes`); - } + // --- Ensure S3 bucket exists once for the batch --- + const s3ForDb = resolveS3ForDatabase(options, storageConfig, databaseId); + await ensureS3BucketExists(options, s3ForDb.bucket, bucket, databaseId, storageConfig.allowedOrigins); - const s3Key = buildS3Key(contentHash); - - // --- Dedup check: look for existing file with same key (content hash) in this bucket --- - const dedupResult = await txClient.query({ - text: `SELECT id - FROM ${storageConfig.filesQualifiedName} - WHERE key = $1 - AND bucket_id = $2 - LIMIT 1`, - values: [s3Key, bucket.id], - }); - - if (dedupResult.rows.length > 0) { - const existingFile = dedupResult.rows[0]; - log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); - - return { - uploadUrl: null as string | null, - fileId: existingFile.id as string, - key: s3Key, - deduplicated: true, - expiresAt: null as string | null, + // --- Process each file --- + const results = []; + for (let i = 0; i < files.length; i++) { + const fileInput = files[i]; + const singleInput = { + ...fileInput, + bucketKey, + ownerId, }; + const result = await processSingleFile( + options, txClient, storageConfig, databaseId, bucket, s3ForDb, singleInput, + ); + results.push({ ...result, index: i }); } - // --- Create file record --- - // For app-level storage (no owner_id column), omit owner_id from the INSERT. - const hasOwnerColumn = storageConfig.membershipType !== null; - const fileResult = await txClient.query({ - text: hasOwnerColumn - ? `INSERT INTO ${storageConfig.filesQualifiedName} - (bucket_id, key, mime_type, size, filename, owner_id, is_public) - VALUES ($1, $2, $3, $4, $5, $6, $7) - RETURNING id` - : `INSERT INTO ${storageConfig.filesQualifiedName} - (bucket_id, key, mime_type, size, filename, is_public) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id`, - values: hasOwnerColumn - ? [ - bucket.id, - s3Key, - contentType, - size, - filename || null, - bucket.owner_id, - bucket.is_public, - ] - : [ - bucket.id, - s3Key, - contentType, - size, - filename || null, - bucket.is_public, - ], - }); - - const fileId = fileResult.rows[0].id; - - // --- Ensure the S3 bucket exists (lazy provisioning) --- - const s3ForDb = resolveS3ForDatabase(options, storageConfig, databaseId); - await ensureS3BucketExists(options, s3ForDb.bucket, bucket, databaseId, storageConfig.allowedOrigins); - - // --- Generate presigned PUT URL (per-database bucket) --- - const uploadUrl = await generatePresignedPutUrl( - s3ForDb, - s3Key, - contentType, - size, - storageConfig.uploadUrlExpirySeconds, - ); - - const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString(); - - return { - uploadUrl, - fileId, - key: s3Key, - deduplicated: false, - expiresAt, - }; + return { files: results }; }); }); }); @@ -362,5 +383,257 @@ export function createPresignedUrlPlugin( })); } +// --- Shared upload logic --- + +/** + * Process a single upload request (used by both requestUploadUrl and requestBulkUploadUrls). + */ +async function processUpload( + options: PresignedUrlPluginOptions, + input: any, + withPgClient: any, + pgSettings: any, +) { + const { bucketKey, ownerId, contentHash, contentType, size, filename, key: customKey } = input; + + if (!bucketKey || typeof bucketKey !== 'string' || bucketKey.length > MAX_BUCKET_KEY_LENGTH) { + throw new Error('INVALID_BUCKET_KEY'); + } + if (!contentHash || typeof contentHash !== 'string' || contentHash.length > MAX_CONTENT_HASH_LENGTH) { + throw new Error('INVALID_CONTENT_HASH'); + } + if (!isValidSha256(contentHash)) { + throw new Error('INVALID_CONTENT_HASH_FORMAT: must be a 64-char lowercase hex SHA-256'); + } + if (!contentType || typeof contentType !== 'string' || contentType.length > MAX_CONTENT_TYPE_LENGTH) { + throw new Error('INVALID_CONTENT_TYPE'); + } + + return withPgClient(pgSettings, async (pgClient: any) => { + return pgClient.withTransaction(async (txClient: any) => { + const databaseId = await resolveDatabaseId(txClient); + if (!databaseId) { + throw new Error('DATABASE_NOT_FOUND'); + } + + const storageConfig = ownerId + ? await getStorageModuleConfigForOwner(txClient, databaseId, ownerId) + : await getStorageModuleConfig(txClient, databaseId); + if (!storageConfig) { + throw new Error( + ownerId + ? 'STORAGE_MODULE_NOT_FOUND_FOR_OWNER: no storage module found for the given ownerId' + : 'STORAGE_MODULE_NOT_PROVISIONED', + ); + } + + if (typeof size !== 'number' || size <= 0 || size > storageConfig.defaultMaxFileSize) { + throw new Error(`INVALID_FILE_SIZE: must be between 1 and ${storageConfig.defaultMaxFileSize} bytes`); + } + if (filename !== undefined && filename !== null) { + if (typeof filename !== 'string' || filename.length > storageConfig.maxFilenameLength) { + throw new Error('INVALID_FILENAME'); + } + } + + const bucket = await getBucketConfig(txClient, storageConfig, databaseId, bucketKey, ownerId); + if (!bucket) { + throw new Error('BUCKET_NOT_FOUND'); + } + + const s3ForDb = resolveS3ForDatabase(options, storageConfig, databaseId); + await ensureS3BucketExists(options, s3ForDb.bucket, bucket, databaseId, storageConfig.allowedOrigins); + + return processSingleFile(options, txClient, storageConfig, databaseId, bucket, s3ForDb, input); + }); + }); +} + +/** + * Process a single file upload within an already-resolved context. + * Handles dedup, custom keys, versioning, and auto-path derivation. + */ +async function processSingleFile( + options: PresignedUrlPluginOptions, + txClient: any, + storageConfig: StorageModuleConfig, + databaseId: string, + bucket: BucketConfig, + s3ForDb: S3Config, + input: any, +) { + const { contentHash, contentType, size, filename, key: customKey } = input; + + // --- Validate inputs --- + if (!contentHash || !isValidSha256(contentHash)) { + throw new Error('INVALID_CONTENT_HASH_FORMAT: must be a 64-char lowercase hex SHA-256'); + } + if (!contentType || typeof contentType !== 'string' || contentType.length > MAX_CONTENT_TYPE_LENGTH) { + throw new Error('INVALID_CONTENT_TYPE'); + } + if (typeof size !== 'number' || size <= 0 || size > storageConfig.defaultMaxFileSize) { + throw new Error(`INVALID_FILE_SIZE: must be between 1 and ${storageConfig.defaultMaxFileSize} bytes`); + } + if (filename !== undefined && filename !== null) { + if (typeof filename !== 'string' || filename.length > storageConfig.maxFilenameLength) { + throw new Error('INVALID_FILENAME'); + } + } + + // --- Validate content type against bucket's allowed_mime_types --- + if (bucket.allowed_mime_types && bucket.allowed_mime_types.length > 0) { + const allowed = bucket.allowed_mime_types as string[]; + const isAllowed = allowed.some((pattern: string) => { + if (pattern === '*/*') return true; + if (pattern.endsWith('/*')) { + const prefix = pattern.slice(0, -1); + return contentType.startsWith(prefix); + } + return contentType === pattern; + }); + if (!isAllowed) { + throw new Error(`CONTENT_TYPE_NOT_ALLOWED: ${contentType} not in bucket allowed types`); + } + } + + // --- Validate size against bucket's max_file_size --- + if (bucket.max_file_size && size > bucket.max_file_size) { + throw new Error(`FILE_TOO_LARGE: exceeds bucket max of ${bucket.max_file_size} bytes`); + } + + // --- Determine S3 key --- + let s3Key: string; + let isCustomKey = false; + if (customKey) { + if (!bucket.allow_custom_keys) { + throw new Error('CUSTOM_KEY_NOT_ALLOWED: bucket does not allow custom keys'); + } + const keyError = validateCustomKey(customKey); + if (keyError) { + throw new Error(keyError); + } + s3Key = customKey; + isCustomKey = true; + } else { + s3Key = buildS3Key(contentHash); + } + + // --- Dedup / versioning check --- + let previousVersionId: string | null = null; + + if (isCustomKey) { + // Custom key mode: check if a file with this key already exists in this bucket. + // If so, auto-version by linking via previous_version_id. + const existingResult = await txClient.query({ + text: `SELECT id, content_hash + FROM ${storageConfig.filesQualifiedName} + WHERE key = $1 + AND bucket_id = $2 + ORDER BY created_at DESC + LIMIT 1`, + values: [s3Key, bucket.id], + }); + + if (existingResult.rows.length > 0) { + const existing = existingResult.rows[0]; + // Same content hash = true dedup (no new upload needed) + if (existing.content_hash === contentHash) { + log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`); + return { + uploadUrl: null as string | null, + fileId: existing.id as string, + key: s3Key, + deduplicated: true, + expiresAt: null as string | null, + previousVersionId: null as string | null, + }; + } + // Different content = new version + previousVersionId = existing.id; + log.info(`Versioning: new version of key ${s3Key}, previous=${previousVersionId}`); + } + } else { + // Hash-based mode: dedup by content_hash in this bucket + const dedupResult = await txClient.query({ + text: `SELECT id + FROM ${storageConfig.filesQualifiedName} + WHERE content_hash = $1 + AND bucket_id = $2 + LIMIT 1`, + values: [contentHash, bucket.id], + }); + + if (dedupResult.rows.length > 0) { + const existingFile = dedupResult.rows[0]; + log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); + + return { + uploadUrl: null as string | null, + fileId: existingFile.id as string, + key: s3Key, + deduplicated: true, + expiresAt: null as string | null, + previousVersionId: null as string | null, + }; + } + } + + // --- Auto-derive ltree path from custom key directory (only when has_path_shares) --- + const derivedPath = isCustomKey && storageConfig.hasPathShares ? derivePathFromKey(s3Key) : null; + + // --- Create file record --- + const hasOwnerColumn = storageConfig.membershipType !== null; + const columns = ['bucket_id', 'key', 'content_hash', 'mime_type', 'size', 'filename', 'is_public']; + const values: any[] = [bucket.id, s3Key, contentHash, contentType, size, filename || null, bucket.is_public]; + let paramIdx = values.length; + + if (hasOwnerColumn) { + columns.push('owner_id'); + values.push(bucket.owner_id); + paramIdx = values.length; + } + if (previousVersionId) { + columns.push('previous_version_id'); + values.push(previousVersionId); + paramIdx = values.length; + } + if (derivedPath) { + columns.push('path'); + values.push(derivedPath); + paramIdx = values.length; + } + + const placeholders = values.map((_: any, i: number) => `$${i + 1}`).join(', '); + const fileResult = await txClient.query({ + text: `INSERT INTO ${storageConfig.filesQualifiedName} + (${columns.join(', ')}) + VALUES (${placeholders}) + RETURNING id`, + values, + }); + + const fileId = fileResult.rows[0].id; + + // --- Generate presigned PUT URL --- + const uploadUrl = await generatePresignedPutUrl( + s3ForDb, + s3Key, + contentType, + size, + storageConfig.uploadUrlExpirySeconds, + ); + + const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString(); + + return { + uploadUrl, + fileId, + key: s3Key, + deduplicated: false, + expiresAt, + previousVersionId, + }; +} + export const PresignedUrlPlugin = createPresignedUrlPlugin; export default PresignedUrlPlugin; diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index b12ff772b..4ec64e5c4 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -11,6 +11,8 @@ const DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS = 3600; // 1 hour const DEFAULT_MAX_FILE_SIZE = 200 * 1024 * 1024; // 200MB const DEFAULT_MAX_FILENAME_LENGTH = 1024; const DEFAULT_CACHE_TTL_SECONDS = process.env.NODE_ENV === 'development' ? 300 : 3600; +const DEFAULT_MAX_BULK_FILES = 100; +const DEFAULT_MAX_BULK_TOTAL_SIZE = 1073741824; // 1GB const FIVE_MINUTES_MS = 1000 * 60 * 5; const ONE_HOUR_MS = 1000 * 60 * 60; @@ -57,6 +59,9 @@ const APP_STORAGE_MODULE_QUERY = ` sm.default_max_file_size, sm.max_filename_length, sm.cache_ttl_seconds, + sm.max_bulk_files, + sm.max_bulk_total_size, + sm.has_path_shares, NULL AS entity_schema, NULL AS entity_table FROM metaschema_modules_public.storage_module sm @@ -93,6 +98,9 @@ const ALL_STORAGE_MODULES_QUERY = ` sm.default_max_file_size, sm.max_filename_length, sm.cache_ttl_seconds, + sm.max_bulk_files, + sm.max_bulk_total_size, + sm.has_path_shares, es.schema_name AS entity_schema, et.name AS entity_table FROM metaschema_modules_public.storage_module sm @@ -122,6 +130,9 @@ interface StorageModuleRow { default_max_file_size: number | null; max_filename_length: number | null; cache_ttl_seconds: number | null; + max_bulk_files: number | null; + max_bulk_total_size: number | null; + has_path_shares: boolean; entity_schema: string | null; entity_table: string | null; } @@ -152,6 +163,9 @@ function buildConfig(row: StorageModuleRow): StorageModuleConfig { defaultMaxFileSize: row.default_max_file_size ?? DEFAULT_MAX_FILE_SIZE, maxFilenameLength: row.max_filename_length ?? DEFAULT_MAX_FILENAME_LENGTH, cacheTtlSeconds, + hasPathShares: row.has_path_shares ?? false, + maxBulkFiles: row.max_bulk_files ?? DEFAULT_MAX_BULK_FILES, + maxBulkTotalSize: row.max_bulk_total_size ?? DEFAULT_MAX_BULK_TOTAL_SIZE, }; } @@ -362,11 +376,11 @@ export async function getBucketConfig( const hasOwner = ownerId && storageConfig.membershipType !== null; const result = await pgClient.query({ text: hasOwner - ? `SELECT id, key, type, is_public, owner_id, allowed_mime_types, max_file_size + ? `SELECT id, key, type, is_public, owner_id, allowed_mime_types, max_file_size, allow_custom_keys FROM ${storageConfig.bucketsQualifiedName} WHERE key = $1 AND owner_id = $2 LIMIT 1` - : `SELECT id, key, type, is_public, ${storageConfig.membershipType !== null ? 'owner_id,' : ''} allowed_mime_types, max_file_size + : `SELECT id, key, type, is_public, ${storageConfig.membershipType !== null ? 'owner_id,' : ''} allowed_mime_types, max_file_size, allow_custom_keys FROM ${storageConfig.bucketsQualifiedName} WHERE key = $1 LIMIT 1`, @@ -385,6 +399,7 @@ export async function getBucketConfig( owner_id: string | null; allowed_mime_types: string[] | null; max_file_size: number | null; + allow_custom_keys: boolean; }; const config: BucketConfig = { @@ -395,6 +410,7 @@ export async function getBucketConfig( owner_id: row.owner_id ?? null, allowed_mime_types: row.allowed_mime_types, max_file_size: row.max_file_size, + allow_custom_keys: row.allow_custom_keys ?? false, }; bucketCache.set(cacheKey, config); diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index b19948705..dc6e394de 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -11,6 +11,7 @@ export interface BucketConfig { owner_id: string | null; allowed_mime_types: string[] | null; max_file_size: number | null; + allow_custom_keys: boolean; } /** @@ -62,6 +63,15 @@ export interface StorageModuleConfig { maxFilenameLength: number; /** Cache TTL in seconds for this config entry (default: 300 dev / 3600 prod) */ cacheTtlSeconds: number; + /** Whether this storage module uses ltree path + path shares (determines if path column exists on files) */ + hasPathShares: boolean; + + // --- Bulk upload limits --- + + /** Max files per requestBulkUploadUrls batch (default: 100) */ + maxBulkFiles: number; + /** Max total size per bulk upload batch in bytes (default: 1GB) */ + maxBulkTotalSize: number; } /** @@ -85,6 +95,13 @@ export interface RequestUploadUrlInput { size: number; /** Original filename (optional, for display/Content-Disposition) */ filename?: string; + /** + * Custom S3 key for the file (only allowed when bucket has allow_custom_keys=true). + * When omitted, key defaults to contentHash (content-addressed dedup). + * When provided, the file is stored at this key; dedup is bypassed. + * Max 1024 chars. Must not contain path traversal (.. or leading /). + */ + key?: string; } /** @@ -101,6 +118,8 @@ export interface RequestUploadUrlPayload { deduplicated: boolean; /** Presigned URL expiry time (null if deduplicated) */ expiresAt: string | null; + /** ID of the previous version (set when re-uploading to an existing custom key) */ + previousVersionId: string | null; } /** From 87385a9be4510e836157c5d8bc7e3b9a096fc192 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 5 May 2026 11:05:34 +0000 Subject: [PATCH 2/2] fix: update snapshots and test seeds for allow_custom_keys + content_hash columns --- .../seed/simple-seed-storage/schema.sql | 3 + .../seed/simple-seed-storage/setup.sql | 5 + .../schema-snapshot.test.ts.snap | 84 +++++ .../__snapshots__/graphile-test.test.ts.snap | 331 ++++++++++++++++++ 4 files changed, 423 insertions(+) diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql index a1ee86869..b01d81184 100644 --- a/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql @@ -28,6 +28,7 @@ CREATE TABLE IF NOT EXISTS "simple-storage-public".buckets ( owner_id uuid, allowed_mime_types text[] NULL, max_file_size bigint NULL, + allow_custom_keys boolean NOT NULL DEFAULT false, created_at timestamptz DEFAULT now(), updated_at timestamptz DEFAULT now(), UNIQUE (key) @@ -40,11 +41,13 @@ CREATE TABLE IF NOT EXISTS "simple-storage-public".files ( id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), bucket_id uuid NOT NULL REFERENCES "simple-storage-public".buckets(id), key text NOT NULL, + content_hash text NOT NULL, mime_type text NOT NULL, size bigint, filename text, owner_id uuid, is_public boolean NOT NULL DEFAULT false, + previous_version_id uuid REFERENCES "simple-storage-public".files(id), created_at timestamptz DEFAULT now(), updated_at timestamptz DEFAULT now(), UNIQUE (bucket_id, key) diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql index 303ecdcd4..ae0c78cf9 100644 --- a/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql @@ -55,11 +55,16 @@ CREATE TABLE IF NOT EXISTS metaschema_modules_public.storage_module ( public_url_prefix text NULL, provider text NULL, allowed_origins text[] NULL, + restrict_reads boolean NOT NULL DEFAULT false, + has_path_shares boolean NOT NULL DEFAULT false, + path_shares_table_id uuid NULL DEFAULT NULL, upload_url_expiry_seconds integer NULL, download_url_expiry_seconds integer NULL, default_max_file_size bigint NULL, max_filename_length integer NULL, cache_ttl_seconds integer NULL, + max_bulk_files integer NULL, + max_bulk_total_size bigint NULL, CONSTRAINT sm_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE ); diff --git a/graphql/server-test/__tests__/__snapshots__/schema-snapshot.test.ts.snap b/graphql/server-test/__tests__/__snapshots__/schema-snapshot.test.ts.snap index 7968b2330..dec27269b 100644 --- a/graphql/server-test/__tests__/__snapshots__/schema-snapshot.test.ts.snap +++ b/graphql/server-test/__tests__/__snapshots__/schema-snapshot.test.ts.snap @@ -2089,6 +2089,15 @@ input RequestUploadUrlInput { """Original filename (optional, for display and Content-Disposition)""" filename: String + + """ + Custom S3 key (e.g., "reports/2024/Q1.pdf"). + Only allowed when the bucket has allow_custom_keys=true. + When omitted, key defaults to contentHash (content-addressed dedup). + When provided, the file is stored at this key. + Re-uploading to an existing key auto-creates a new version. + """ + key: String } type RequestUploadUrlPayload { @@ -2106,6 +2115,69 @@ type RequestUploadUrlPayload { """Presigned URL expiry time (null if deduplicated)""" expiresAt: Datetime + + """ + ID of the previous version (set when re-uploading to an existing custom key) + """ + previousVersionId: UUID +} + +input BulkUploadFileInput { + """SHA-256 content hash computed by the client (hex-encoded, 64 chars)""" + contentHash: String! + + """MIME type of the file (e.g., "image/png")""" + contentType: String! + + """File size in bytes""" + size: Int! + + """Original filename (optional, for display and Content-Disposition)""" + filename: String + + """Custom S3 key (only when bucket has allow_custom_keys=true)""" + key: String +} + +input RequestBulkUploadUrlsInput { + """Bucket key (e.g., "public", "private")""" + bucketKey: String! + + """Owner entity ID for entity-scoped uploads""" + ownerId: UUID + + """Array of files to upload""" + files: [BulkUploadFileInput!]! +} + +type BulkUploadFilePayload { + """Presigned PUT URL (null if file was deduplicated)""" + uploadUrl: String + + """The file ID""" + fileId: UUID! + + """The S3 object key""" + key: String! + + """Whether this file was deduplicated""" + deduplicated: Boolean! + + """Presigned URL expiry time (null if deduplicated)""" + expiresAt: Datetime + + """ + ID of the previous version (set when re-uploading to an existing custom key) + """ + previousVersionId: UUID + + """Index of this file in the input array (for client correlation)""" + index: Int! +} + +type RequestBulkUploadUrlsPayload { + """Array of results, one per input file""" + files: [BulkUploadFilePayload!]! } """The root query type which gives access points into the data universe.""" @@ -2398,6 +2470,18 @@ type Mutation { input: RequestUploadUrlInput! ): RequestUploadUrlPayload + """ + Request presigned URLs for uploading multiple files in a single batch. + Subject to per-storage-module limits (max_bulk_files, max_bulk_total_size). + Each file is processed independently — some may dedup while others get fresh URLs. + """ + requestBulkUploadUrls( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: RequestBulkUploadUrlsInput! + ): RequestBulkUploadUrlsPayload + """ Provision an S3 bucket for a logical bucket in the database. Reads the bucket config via RLS, then creates and configures diff --git a/graphql/test/__tests__/__snapshots__/graphile-test.test.ts.snap b/graphql/test/__tests__/__snapshots__/graphile-test.test.ts.snap index 57627483c..68edec297 100644 --- a/graphql/test/__tests__/__snapshots__/graphile-test.test.ts.snap +++ b/graphql/test/__tests__/__snapshots__/graphile-test.test.ts.snap @@ -3262,6 +3262,20 @@ that owns this entity instance (e.g., a data room ID, team ID).", "ofType": null, }, }, + { + "defaultValue": null, + "description": "Custom S3 key (e.g., "reports/2024/Q1.pdf"). +Only allowed when the bucket has allow_custom_keys=true. +When omitted, key defaults to contentHash (content-addressed dedup). +When provided, the file is stored at this key. +Re-uploading to an existing key auto-creates a new version.", + "name": "key", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, ], "interfaces": null, "kind": "INPUT_OBJECT", @@ -3354,6 +3368,18 @@ that owns this entity instance (e.g., a data room ID, team ID).", "ofType": null, }, }, + { + "args": [], + "deprecationReason": null, + "description": "ID of the previous version (set when re-uploading to an existing custom key)", + "isDeprecated": false, + "name": "previousVersionId", + "type": { + "kind": "SCALAR", + "name": "UUID", + "ofType": null, + }, + }, ], "inputFields": null, "interfaces": [], @@ -3375,6 +3401,282 @@ to unexpected results.", "name": "Datetime", "possibleTypes": null, }, + { + "description": null, + "enumValues": null, + "fields": null, + "inputFields": [ + { + "defaultValue": null, + "description": "SHA-256 content hash computed by the client (hex-encoded, 64 chars)", + "name": "contentHash", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + }, + { + "defaultValue": null, + "description": "MIME type of the file (e.g., "image/png")", + "name": "contentType", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + }, + { + "defaultValue": null, + "description": "File size in bytes", + "name": "size", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null, + }, + }, + }, + { + "defaultValue": null, + "description": "Original filename (optional, for display and Content-Disposition)", + "name": "filename", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + { + "defaultValue": null, + "description": "Custom S3 key (only when bucket has allow_custom_keys=true)", + "name": "key", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + ], + "interfaces": null, + "kind": "INPUT_OBJECT", + "name": "BulkUploadFileInput", + "possibleTypes": null, + }, + { + "description": null, + "enumValues": null, + "fields": null, + "inputFields": [ + { + "defaultValue": null, + "description": "Bucket key (e.g., "public", "private")", + "name": "bucketKey", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + }, + { + "defaultValue": null, + "description": "Owner entity ID for entity-scoped uploads", + "name": "ownerId", + "type": { + "kind": "SCALAR", + "name": "UUID", + "ofType": null, + }, + }, + { + "defaultValue": null, + "description": "Array of files to upload", + "name": "files", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "BulkUploadFileInput", + "ofType": null, + }, + }, + }, + }, + }, + ], + "interfaces": null, + "kind": "INPUT_OBJECT", + "name": "RequestBulkUploadUrlsInput", + "possibleTypes": null, + }, + { + "description": null, + "enumValues": null, + "fields": [ + { + "args": [], + "deprecationReason": null, + "description": "Presigned PUT URL (null if file was deduplicated)", + "isDeprecated": false, + "name": "uploadUrl", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "The file ID", + "isDeprecated": false, + "name": "fileId", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "UUID", + "ofType": null, + }, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "The S3 object key", + "isDeprecated": false, + "name": "key", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null, + }, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "Whether this file was deduplicated", + "isDeprecated": false, + "name": "deduplicated", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null, + }, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "Presigned URL expiry time (null if deduplicated)", + "isDeprecated": false, + "name": "expiresAt", + "type": { + "kind": "SCALAR", + "name": "Datetime", + "ofType": null, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "ID of the previous version (set when re-uploading to an existing custom key)", + "isDeprecated": false, + "name": "previousVersionId", + "type": { + "kind": "SCALAR", + "name": "UUID", + "ofType": null, + }, + }, + { + "args": [], + "deprecationReason": null, + "description": "Index of this file in the input array (for client correlation)", + "isDeprecated": false, + "name": "index", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null, + }, + }, + }, + ], + "inputFields": null, + "interfaces": [], + "kind": "OBJECT", + "name": "BulkUploadFilePayload", + "possibleTypes": null, + }, + { + "description": null, + "enumValues": null, + "fields": [ + { + "args": [], + "deprecationReason": null, + "description": "Array of results, one per input file", + "isDeprecated": false, + "name": "files", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "BulkUploadFilePayload", + "ofType": null, + }, + }, + }, + }, + }, + ], + "inputFields": null, + "interfaces": [], + "kind": "OBJECT", + "name": "RequestBulkUploadUrlsPayload", + "possibleTypes": null, + }, { "description": "The root query type which gives access points into the data universe.", "enumValues": null, @@ -3605,6 +3907,35 @@ existing file ID and deduplicated=true with no uploadUrl.", "ofType": null, }, }, + { + "args": [ + { + "defaultValue": null, + "description": "The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields.", + "name": "input", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RequestBulkUploadUrlsInput", + "ofType": null, + }, + }, + }, + ], + "deprecationReason": null, + "description": "Request presigned URLs for uploading multiple files in a single batch. +Subject to per-storage-module limits (max_bulk_files, max_bulk_total_size). +Each file is processed independently — some may dedup while others get fresh URLs.", + "isDeprecated": false, + "name": "requestBulkUploadUrls", + "type": { + "kind": "OBJECT", + "name": "RequestBulkUploadUrlsPayload", + "ofType": null, + }, + }, { "args": [ {