From e654546519ad5f58da4d67e98db236b8c238cf77 Mon Sep 17 00:00:00 2001 From: BLxcwg666 Date: Sat, 10 Jan 2026 03:09:55 +0800 Subject: [PATCH] feat: refactor S3 upload and add configurable options - Add S3 upload configuration options - Add path placeholder utility for file organization - Add upload size configuration - Implement image type support in file management - Support local and S3 storage deletion - Add uuid dependency for unique file naming --- apps/core/package.json | 1 + .../core/src/modules/backup/backup.service.ts | 38 ++-- .../src/modules/configs/configs.default.ts | 23 ++- apps/core/src/modules/configs/configs.dto.ts | 124 ++++++++++--- .../src/modules/configs/configs.interface.ts | 9 + .../src/modules/configs/configs.service.ts | 48 +++-- apps/core/src/modules/file/file.controller.ts | 42 ++++- apps/core/src/modules/file/file.dto.ts | 12 +- apps/core/src/modules/file/file.service.ts | 167 +++++++++++++++++- .../controllers/base.option.controller.ts | 19 +- .../helper/helper.upload.service.ts | 16 ++ apps/core/src/utils/path-placeholder.util.ts | 162 +++++++++++++++++ apps/core/src/utils/s3.util.ts | 78 +++++++- pnpm-lock.yaml | 9 + 14 files changed, 687 insertions(+), 61 deletions(-) create mode 100644 apps/core/src/utils/path-placeholder.util.ts diff --git a/apps/core/package.json b/apps/core/package.json index da6049d4f2d..de68290a582 100644 --- a/apps/core/package.json +++ b/apps/core/package.json @@ -127,6 +127,7 @@ "snakecase-keys": "6.0.0", "source-map-support": "^0.5.21", "ua-parser-js": "2.0.6", + "uuid": "^13.0.0", "vm2": "3.9.19", "wildcard-match": "5.1.4", "xss": "1.0.15", diff --git a/apps/core/src/modules/backup/backup.service.ts b/apps/core/src/modules/backup/backup.service.ts index 27ac9e1ed47..cb177d29dfa 100644 --- a/apps/core/src/modules/backup/backup.service.ts +++ b/apps/core/src/modules/backup/backup.service.ts @@ -279,7 +279,7 @@ export class BackupService { } @CronOnce(CronExpression.EVERY_DAY_AT_1AM, { name: 'backupDB' }) - @CronDescription('备份 DB 并上传 COS') + @CronDescription('备份 DB 并上传 S3') async backupDB() { const backup = await this.backup() if (!backup) { @@ -288,26 +288,44 @@ export class BackupService { } scheduleManager.schedule(async () => { - const { backupOptions } = await this.configs.waitForConfigReady() + const { backupOptions, s3Options } = + await this.configs.waitForConfigReady() - const { endpoint, bucket, region, secretId, secretKey } = - backupOptions || {} - if (!endpoint || !bucket || !region || !secretId || !secretKey) { + if (!backupOptions?.enable) { + this.logger.log('未启用 S3 备份') + return + } + + const { endpoint, bucket, region, accessKeyId, secretAccessKey } = + s3Options || {} + if (!endpoint || !bucket || !region || !accessKeyId || !secretAccessKey) { + this.logger.warn('S3 配置不完整，无法上传备份') return } const s3 = new S3Uploader({ bucket, region, - accessKey: secretId, - secretKey, + accessKey: accessKeyId, + secretKey: secretAccessKey, endpoint, }) - const remoteFileKey = backup.path.slice(backup.path.lastIndexOf('/') + 1) - this.logger.log('--> 开始上传到 S3') + if (s3Options.customDomain) { + s3.setCustomDomain(s3Options.customDomain) + } + + const { parsePlaceholder } = await import('~/utils/path-placeholder.util') + const localFilename = backup.path.slice(backup.path.lastIndexOf('/') + 1) + const remotePath = backupOptions.path || 'backups/{Y}/{m}/{filename}' + const remoteFileKey = parsePlaceholder(remotePath, { + filename: localFilename, + }) + + this.logger.log(`--> 开始上传到 S3: ${remoteFileKey}`) await s3.uploadFile(backup.buffer, remoteFileKey).catch((error) => { - this.logger.error('--> 上传失败了') + this.logger.error('--> 上传失败') + this.logger.error(error) throw error }) diff --git a/apps/core/src/modules/configs/configs.default.ts b/apps/core/src/modules/configs/configs.default.ts index ab3a1cf34c5..ee88b109d29 100644 --- a/apps/core/src/modules/configs/configs.default.ts +++ b/apps/core/src/modules/configs/configs.default.ts @@ -48,13 +48,24 @@ export const generateDefaultConfig: () => IConfig = () => ({ allowSubPath: false, enableAvatarInternalization: true, }, + s3Options: { + endpoint: '', + accessKeyId: '', + secretAccessKey: '', + bucket: '', + region: '', + customDomain: '', + pathStyleAccess: false, + }, backupOptions: { - enable: true, - endpoint: null!, - region: null!, - bucket: null!, - secretId: null!, - secretKey: null!, + enable: false, + path: 'backups/{Y}/{m}/backup-{Y}{m}{d}-{h}{i}{s}.zip', + }, + imageBedOptions: { + enable: false, + path: 'images/{Y}/{m}/{uuid}.{ext}', + allowedFormats: 'jpg,jpeg,png,gif,webp', + maxSizeMB: 10, }, baiduSearchOptions: { enable: false, token: null! }, bingSearchOptions: { enable: false, token: null! }, diff --git a/apps/core/src/modules/configs/configs.dto.ts b/apps/core/src/modules/configs/configs.dto.ts index f6f1b4afab1..1ee9a9da43b 100644 --- a/apps/core/src/modules/configs/configs.dto.ts +++ b/apps/core/src/modules/configs/configs.dto.ts @@ -76,9 +76,15 @@ export class UrlDto { } class MailOption { - @IsInt() - @Transform(({ value: val }) => Number.parseInt(val)) @IsOptional() + @Transform( + ({ value }) => { + if (value === undefined || value === null) return undefined + return typeof value === 'number' ? value : Number.parseInt(value, 10) + }, + { toClassOnly: true }, + ) + @IsInt() @JSONSchemaNumberField('SMTP 端口', halfFieldOption) port: number @IsUrl({ require_protocol: false }) @@ -150,11 +156,17 @@ export class CommentOptionsDto { }) aiReviewType: 'binary' | 'score' + @IsOptional() + @Transform( + ({ value }) => { + if (value === undefined || value === null) return undefined + return typeof value === 'number' ? value : Number.parseInt(value, 10) + }, + { toClassOnly: true }, + ) @IsInt() - @Transform(({ value: val }) => Number.parseInt(val)) @Min(1) @Max(10) - @IsOptional() @JSONSchemaNumberField('AI 审核阈值', { description: '分数大于多少时会被归类为垃圾评论，范围为 1-10, 默认为 5', }) @@ -193,40 +205,112 @@ export class CommentOptionsDto { recordIpLocation?: boolean } -@JSONSchema({ title: '备份' }) -export class BackupOptionsDto { - @IsBoolean() - @IsOptional() - @JSONSchemaToggleField('开启自动备份', { - description: '填写以下 S3 信息，将同时上传备份到 S3', - }) - enable: boolean - +@JSONSchema({ title: 'S3 对象存储设置' }) +export class S3OptionsDto { @IsString() @IsOptional() - @JSONSchemaPlainField('S3 服务端点') + @JSONSchemaPlainField('S3 服务端点', { + description: + '例如：https://s3.amazonaws.com 或 https://oss-cn-hangzhou.aliyuncs.com', + }) endpoint?: string @IsString() @IsOptional() - @JSONSchemaHalfGirdPlainField('SecretId') - secretId?: string + @JSONSchemaHalfGirdPlainField('Access Key ID / SecretId') + accessKeyId?: string @IsOptional() @IsString() - @JSONSchemaPasswordField('SecretKey', halfFieldOption) + @JSONSchemaPasswordField('Secret Access Key / SecretKey', halfFieldOption) @SecretField - secretKey?: string + secretAccessKey?: string @IsOptional() @IsString() - @JSONSchemaHalfGirdPlainField('Bucket') + @JSONSchemaHalfGirdPlainField('Bucket 名称') bucket?: string @IsString() @IsOptional() @JSONSchemaHalfGirdPlainField('地域 Region') - region: string + region?: string + + @IsString() + @IsOptional() + @JSONSchemaPlainField('自定义域名', { + description: + '如果配置了 CDN 或自定义域名，填写此项；留空则使用默认的 S3 URL', + }) + customDomain?: string + + @IsBoolean() + @IsOptional() + @JSONSchemaToggleField('路径风格访问', { + description: + '启用路径风格访问（Path-style），适用于 MinIO 等兼容 S3 的服务', + }) + pathStyleAccess?: boolean +} + +@JSONSchema({ title: '备份设置' }) +export class BackupOptionsDto { + @IsBoolean() + @IsOptional() + @JSONSchemaToggleField('开启自动备份到 S3', { + description: '需要先配置 S3 对象存储设置', + }) + enable: boolean + + @IsString() + @IsOptional() + @JSONSchemaPlainField('备份文件路径', { + description: + '支持占位符：{Y}年4位 {y}年2位 {m}月 {d}日 {h}时 {i}分 {s}秒 {timestamp}时间戳 {uuid} {md5} 等', + }) + path?: string +} + +@JSONSchema({ title: '图床设置' }) +export class ImageBedOptionsDto { + @IsBoolean() + @IsOptional() + @JSONSchemaToggleField('启用 S3 图床', { + description: + '启用后，编辑器上传的图片将存储到 S3；需要先配置 S3 对象存储设置', + }) + enable: boolean + + @IsString() + @IsOptional() + @JSONSchemaPlainField('图片存储路径', { + description: + '支持占位符：{Y}年4位 {y}年2位 {m}月 {d}日 {h}时 {i}分 {s}秒 {timestamp}时间戳 {uuid} {md5} {md5-16} {str-N}随机字符串 {filename}原文件名', + }) + path?: string + + @IsString() + @IsOptional() + @JSONSchemaPlainField('允许的图片格式', { + description: '逗号分隔，例如：jpg,jpeg,png,gif,webp', + }) + allowedFormats?: string + + @IsOptional() + @Transform( + ({ value }) => { + if (value === undefined || value === null) return undefined + return typeof value === 'number' ? value : Number.parseInt(value, 10) + }, + { toClassOnly: true }, + ) + @IsInt() + @Min(1) + @Max(100) + @JSONSchemaNumberField('最大文件大小（MB）', { + description: '单个图片文件的最大大小限制，单位：MB', + }) + maxSizeMB?: number } @JSONSchema({ title: '百度推送设定' }) diff --git a/apps/core/src/modules/configs/configs.interface.ts b/apps/core/src/modules/configs/configs.interface.ts index dbd86d30f6e..e4d202d665f 100644 --- a/apps/core/src/modules/configs/configs.interface.ts +++ b/apps/core/src/modules/configs/configs.interface.ts @@ -18,8 +18,10 @@ import { CommentOptionsDto, FeatureListDto, FriendLinkOptionsDto, + ImageBedOptionsDto, MailOptionsDto, OAuthDto, + S3OptionsDto, SeoDto, TextOptionsDto, ThirdPartyServiceIntegrationDto, @@ -63,8 +65,15 @@ export abstract class IConfig { @ConfigField(() => FriendLinkOptionsDto) friendLinkOptions: Required + @ConfigField(() => S3OptionsDto) + s3Options: Required + @ConfigField(() => BackupOptionsDto) backupOptions: Required + + @ConfigField(() => ImageBedOptionsDto) + imageBedOptions: Required + @ConfigField(() => BaiduSearchOptionsDto) baiduSearchOptions: Required @ConfigField(() => BingSearchOptionsDto) diff --git a/apps/core/src/modules/configs/configs.service.ts b/apps/core/src/modules/configs/configs.service.ts index bbffe07c51e..754c9875754 100644 --- a/apps/core/src/modules/configs/configs.service.ts +++ b/apps/core/src/modules/configs/configs.service.ts @@ -140,25 +140,53 @@ export class ConfigsService { } } + private removeUndefinedValues(obj: T): T { + if (obj === null || typeof obj !== 'object') { + return obj + } + + if (Array.isArray(obj)) { + return obj as T + } + + const result = {} as T + for (const [key, value] of Object.entries(obj)) { + if (value !== undefined) { + result[key as keyof T] = + typeof value === 'object' && value !== null + ? this.removeUndefinedValues(value) + : value + } + } + return result + } + private async patch( key: T, data: Partial, ): Promise { const config = await this.getConfig() + + const filteredData = this.removeUndefinedValues(data) + const updatedConfigRow = await this.optionModel .findOneAndUpdate( { name: key as string }, { - value: mergeWith(cloneDeep(config[key]), data, (old, newer) => { - // 数组不合并 - if (Array.isArray(old)) { - return newer - } - // 对象合并 - if (typeof old === 'object' && typeof newer === 'object') { - return { ...old, ...newer } - } - }), + value: mergeWith( + cloneDeep(config[key]), + filteredData, + (old, newer) => { + // 数组不合并 + if (Array.isArray(old)) { + return newer + } + // 对象合并 + if (typeof old === 'object' && typeof newer === 'object') { + return { ...old, ...newer } + } + }, + ), }, { upsert: true, new: true }, ) diff --git a/apps/core/src/modules/file/file.controller.ts b/apps/core/src/modules/file/file.controller.ts index a0be873945e..b85901c7453 100644 --- a/apps/core/src/modules/file/file.controller.ts +++ b/apps/core/src/modules/file/file.controller.ts @@ -22,7 +22,12 @@ import { UploadService } from '~/processors/helper/helper.upload.service' import { PagerDto } from '~/shared/dto/pager.dto' import { FastifyReply, FastifyRequest } from 'fastify' import { lookup } from 'mime-types' -import { FileQueryDto, FileUploadDto, RenameFileQueryDto } from './file.dto' +import { + FileDeleteQueryDto, + FileQueryDto, + FileUploadDto, + RenameFileQueryDto, +} from './file.dto' import { FileService } from './file.service' const { customAlphabet } = nanoid @@ -95,19 +100,50 @@ export class FileController { const ext = path.extname(file.filename) const filename = customAlphabet(alphabet)(18) + ext.toLowerCase() + const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'] + const isImage = imageExtensions.includes(ext.toLowerCase()) + + if (isImage && type === 'photo') { + const buffer = await this.uploadService.getFileBuffer(file.file) + + await this.service.validateImageFile(filename, buffer) + const s3Url = await this.service.uploadImageToS3(filename, buffer) + + if (s3Url) { + return { + url: s3Url, + name: filename, + storage: 's3', + } + } + + await this.service.writeFileFromBuffer(type, filename, buffer) + + return { + url: await this.service.resolveFileUrl(type, filename), + name: filename, + storage: 'local', + } + } + await this.service.writeFile(type, filename, file.file) return { url: await this.service.resolveFileUrl(type, filename), name: filename, + storage: 'local', } } @Delete('/:type/:name') @Auth() - async delete(@Param() params: FileQueryDto) { + async delete( + @Param() params: FileQueryDto, + @Query() query: FileDeleteQueryDto, + ) { const { type, name } = params - await this.service.deleteFile(type, name) + const { storage, url } = query + await this.service.deleteFile(type, name, storage, url) } @Auth() diff --git a/apps/core/src/modules/file/file.dto.ts b/apps/core/src/modules/file/file.dto.ts index d06324fef3a..011eca77038 100644 --- a/apps/core/src/modules/file/file.dto.ts +++ b/apps/core/src/modules/file/file.dto.ts @@ -1,4 +1,4 @@ -import { IsEnum, IsOptional, IsString } from 'class-validator' +import { IsEnum, IsIn, IsOptional, IsString } from 'class-validator' import { FileType, FileTypeEnum } from './file.type' export class FileQueryDto { @@ -8,6 +8,16 @@ export class FileQueryDto { name: string } +export class FileDeleteQueryDto { + @IsOptional() + @IsIn(['local', 's3']) + storage?: 'local' | 's3' + + @IsOptional() + @IsString() + url?: string +} + export class FileUploadDto { @IsEnum(FileTypeEnum) @IsOptional() diff --git a/apps/core/src/modules/file/file.service.ts b/apps/core/src/modules/file/file.service.ts index e1487f085f8..3530baeb236 100644 --- a/apps/core/src/modules/file/file.service.ts +++ b/apps/core/src/modules/file/file.service.ts @@ -13,6 +13,8 @@ import { STATIC_FILE_DIR, STATIC_FILE_TRASH_DIR, } from '~/constants/path.constant' +import { parsePlaceholder } from '~/utils/path-placeholder.util' +import { S3Uploader } from '~/utils/s3.util' import { ConfigsService } from '../configs/configs.service' import type { FileType } from './file.type' @@ -23,6 +25,94 @@ export class FileService { this.logger = new Logger(FileService.name) } + async validateImageFile(filename: string, buffer: Buffer): Promise { + const { imageBedOptions } = await this.configService.waitForConfigReady() + + const ext = path.extname(filename).slice(1).toLowerCase() + const allowedFormats = imageBedOptions?.allowedFormats + ?.split(',') + .map((f) => f.trim().toLowerCase()) + if ( + allowedFormats && + allowedFormats.length > 0 && + !allowedFormats.includes(ext) + ) { + throw new BadRequestException( + `不支持的图片格式: ${ext}，允许的格式: ${imageBedOptions.allowedFormats}`, + ) + } + + const maxSizeMB = imageBedOptions?.maxSizeMB || 10 + const maxSizeBytes = maxSizeMB * 1024 * 1024 + if (buffer.length > maxSizeBytes) { + throw new BadRequestException( + `图片文件过大: ${(buffer.length / 1024 / 1024).toFixed(2)}MB，最大允许: ${maxSizeMB}MB`, + ) + } + } + + async uploadImageToS3( + filename: string, + buffer: Buffer, + ): Promise { + const { imageBedOptions, s3Options } = + await this.configService.waitForConfigReady() + + if (!imageBedOptions?.enable) { + return null + } + + const { endpoint, bucket, region, accessKeyId, secretAccessKey } = + s3Options || {} + if (!endpoint || !bucket || !region || !accessKeyId || !secretAccessKey) { + this.logger.warn('S3 配置不完整，无法上传图片到 S3') + return null + } + + const ext = path.extname(filename).slice(1).toLowerCase() + + const s3 = new S3Uploader({ + bucket, + region, + accessKey: accessKeyId, + secretKey: secretAccessKey, + endpoint, + }) + + if (s3Options.customDomain) { + s3.setCustomDomain(s3Options.customDomain) + } + + const pathTemplate = imageBedOptions.path || 'images/{Y}/{m}/{uuid}.{ext}' + const remotePath = parsePlaceholder(pathTemplate, { + filename, + }) + + try { + const mimeTypes: Record = { + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + png: 'image/png', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + } + const contentType = mimeTypes[ext] || 'application/octet-stream' + + this.logger.log(`Uploading to S3: ${remotePath}`) + await s3.uploadToS3(remotePath, buffer, contentType) + this.logger.log(`Successfully uploaded to S3: ${remotePath}`) + + const baseUrl = s3Options.customDomain || endpoint + return `${baseUrl.replace(/\/+$/, '')}/${remotePath}` + } catch (error) { + this.logger.error('Failed to upload to s3', error) + throw new InternalServerErrorException( + `上传图片到 S3 失败: ${error.message}`, + ) + } + } + private resolveFilePath(type: FileType, name: string) { return path.resolve(STATIC_FILE_DIR, type, name) } @@ -74,11 +164,33 @@ export class FileService { }) } - async deleteFile(type: FileType, name: string) { + async writeFileFromBuffer( + type: FileType, + name: string, + buffer: Buffer, + ): Promise { + const filePath = this.resolveFilePath(type, name) + if (await this.checkIsExist(filePath)) { + throw new BadRequestException('文件已存在') + } + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile(filePath, buffer) + } + + async deleteFile( + type: FileType, + name: string, + storage?: 'local' | 's3', + url?: string, + ) { try { - const path = this.resolveFilePath(type, name) - await fs.copyFile(path, resolve(STATIC_FILE_TRASH_DIR, name)) - await fs.unlink(path) + if (storage === 's3') { + await this.deleteFileFromS3(url) + } else { + const path = this.resolveFilePath(type, name) + await fs.copyFile(path, resolve(STATIC_FILE_TRASH_DIR, name)) + await fs.unlink(path) + } } catch (error) { this.logger.error('删除文件失败', error) @@ -86,6 +198,53 @@ export class FileService { } } + async deleteFileFromS3(fileUrl?: string): Promise { + if (!fileUrl) { + throw new BadRequestException('删除 S3 文件需要提供 URL') + } + + const { imageBedOptions, s3Options } = + await this.configService.waitForConfigReady() + + if (!imageBedOptions?.enable) { + throw new InternalServerErrorException('S3 图床未启用') + } + + const { endpoint, bucket, region, accessKeyId, secretAccessKey } = + s3Options || {} + if (!endpoint || !bucket || !region || !accessKeyId || !secretAccessKey) { + throw new InternalServerErrorException('S3 配置不完整') + } + + const s3 = new S3Uploader({ + bucket, + region, + accessKey: accessKeyId, + secretKey: secretAccessKey, + endpoint, + }) + + let objectKey: string + try { + const urlObj = new URL(fileUrl) + objectKey = urlObj.pathname.replace(/^\//, '') // 移除开头的斜杠 + } catch (error) { + this.logger.error('解析 S3 URL 失败', error) + throw new BadRequestException(`无效的 S3 URL: ${fileUrl}`) + } + + try { + this.logger.log(`Deleting from S3: ${objectKey}`) + await s3.deleteFromS3(objectKey) + this.logger.log(`Successfully deleted from S3: ${objectKey}`) + } catch (error) { + this.logger.error('Failed to delete from S3', error) + throw new InternalServerErrorException( + `从 S3 删除文件失败: ${error.message}`, + ) + } + } + async getDir(type: FileType) { await fs.mkdir(this.resolveFilePath(type, ''), { recursive: true }) const path_1 = path.resolve(STATIC_FILE_DIR, type) diff --git a/apps/core/src/modules/option/controllers/base.option.controller.ts b/apps/core/src/modules/option/controllers/base.option.controller.ts index 79ca8f087a1..22b0383f7c3 100644 --- a/apps/core/src/modules/option/controllers/base.option.controller.ts +++ b/apps/core/src/modules/option/controllers/base.option.controller.ts @@ -10,7 +10,6 @@ import { HTTPDecorators } from '~/common/decorators/http.decorator' import { IConfig } from '~/modules/configs/configs.interface' import { ConfigsService } from '~/modules/configs/configs.service' import { classToJsonSchema } from '~/utils/jsonschema.util' -import { instanceToPlain } from 'class-transformer' import { ConfigKeyDto } from '../dtoes/config.dto' import { OptionController } from '../option.decorator' @@ -18,9 +17,11 @@ import { OptionController } from '../option.decorator' export class BaseOptionController { constructor(private readonly configsService: ConfigsService) {} + @HTTPDecorators.Bypass @Get('/') - getOption() { - return instanceToPlain(this.configsService.getConfig()) + async getOption() { + const config = await this.configsService.getConfig() + return JSON.parse(JSON.stringify(config)) } @HTTPDecorators.Bypass @@ -31,6 +32,7 @@ export class BaseOptionController { }) } + @HTTPDecorators.Bypass @Get('/:key') async getOptionKey(@Param('key') key: keyof IConfig) { if (typeof key !== 'string' && !key) { @@ -42,14 +44,19 @@ export class BaseOptionController { if (!value) { throw new BadRequestException('key is not exists.') } - return { data: instanceToPlain(value) } + return { data: JSON.parse(JSON.stringify(value)) } } + @HTTPDecorators.Bypass @Patch('/:key') - patch(@Param() params: ConfigKeyDto, @Body() body: Record) { + async patch( + @Param() params: ConfigKeyDto, + @Body() body: Record, + ) { if (typeof body !== 'object') { throw new UnprocessableEntityException('body must be object') } - return this.configsService.patchAndValid(params.key, body) + const result = await this.configsService.patchAndValid(params.key, body) + return JSON.parse(JSON.stringify(result)) } } diff --git a/apps/core/src/processors/helper/helper.upload.service.ts b/apps/core/src/processors/helper/helper.upload.service.ts index bf2f71d974c..2be5d4568bd 100644 --- a/apps/core/src/processors/helper/helper.upload.service.ts +++ b/apps/core/src/processors/helper/helper.upload.service.ts @@ -1,3 +1,4 @@ +import type { Readable } from 'node:stream' import type { MultipartFile } from '@fastify/multipart' import { BadRequestException, Injectable } from '@nestjs/common' import type { FastifyRequest } from 'fastify' @@ -26,4 +27,19 @@ export class UploadService { return data } + + public async getFileBuffer(stream: Readable): Promise { + const chunks: Buffer[] = [] + return new Promise((resolve, reject) => { + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + stream.on('end', () => { + resolve(Buffer.concat(chunks)) + }) + stream.on('error', (error) => { + reject(error) + }) + }) + } } diff --git a/apps/core/src/utils/path-placeholder.util.ts b/apps/core/src/utils/path-placeholder.util.ts new file mode 100644 index 00000000000..f7c95c641ff --- /dev/null +++ b/apps/core/src/utils/path-placeholder.util.ts @@ -0,0 +1,162 @@ +import { createHash, randomBytes } from 'node:crypto' +import { extname } from 'node:path' +import { v4 as uuidv4 } from 'uuid' + +/** + * {Y} - 年份 4位 + * {y} - 年份 2位 + * {m} - 月份 2位 + * {d} - 日期 2位 + * {h} - 小时 2位 + * {i} - 分钟 2位 + * {s} - 秒钟 2位 + * {ms} - 毫秒 3位 + * {timestamp} - 时间戳（毫秒） + * {md5} - 随机MD5字符串（32位） + * {md5-16} - 随机MD5字符串（16位） + * {uuid} - UUID字符串 + * {str-N} - 随机字符串，N为数字，表示字符串的长度，例如 {str-8} + * {filename} - 原文件名（不含扩展名） + * {ext} - 文件扩展名（不含点） + */ + +interface PlaceholderOptions { + filename?: string + date?: Date +} + +function generateRandomString(length: number): string { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + let result = '' + const bytes = randomBytes(length) + for (let i = 0; i < length; i++) { + result += chars[bytes[i] % chars.length] + } + return result +} + +function generateMD5(length?: number): string { + const hash = createHash('md5').update(randomBytes(16)).digest('hex') + return length ? hash.slice(0, Math.max(0, length)) : hash +} + +function pad(num: number, size: number): string { + return String(num).padStart(size, '0') +} + +/** + * 解析路径占位符 + * @param template 包含占位符的路径模板 + * @param options 选项 + * @returns 解析后的路径 + */ +export function parsePlaceholder( + template: string, + options: PlaceholderOptions = {}, +): string { + const { filename, date = new Date() } = options + + let baseFilename = '' + let ext = '' + if (filename) { + ext = extname(filename).slice(1) + baseFilename = filename.slice(0, filename.length - ext.length - 1) + } + + const year4 = date.getFullYear() + const year2 = String(year4).slice(-2) + const month = pad(date.getMonth() + 1, 2) + const day = pad(date.getDate(), 2) + const hour = pad(date.getHours(), 2) + const minute = pad(date.getMinutes(), 2) + const second = pad(date.getSeconds(), 2) + const millisecond = pad(date.getMilliseconds(), 3) + const timestamp = date.getTime() + + let result = template + + result = result + .replaceAll('{Y}', String(year4)) + .replaceAll('{y}', year2) + .replaceAll('{m}', month) + .replaceAll('{d}', day) + .replaceAll('{h}', hour) + .replaceAll('{i}', minute) + .replaceAll('{s}', second) + .replaceAll('{ms}', millisecond) + .replaceAll('{timestamp}', String(timestamp)) + .replaceAll('{filename}', baseFilename) + .replaceAll('{ext}', ext) + .replaceAll('{uuid}', () => uuidv4()) + .replaceAll('{md5-16}', () => generateMD5(16)) + .replaceAll('{md5}', () => generateMD5()) + .replaceAll(/\{str-(\d+)\}/g, (_, length) => { + return generateRandomString(Number.parseInt(length)) + }) + + return result +} + +/** + * 验证路径模板是否有效 + * @param template 路径模板 + * @returns 是否有效 + */ +export function validatePlaceholderTemplate(template: string): { + valid: boolean + errors: string[] +} { + const errors: string[] = [] + + // fuck eslint + const illegalCharsPattern = /[<>"|?*]/ + const hasControlChars = [...template].some((char) => { + const code = char.charCodeAt(0) + return code >= 0 && code <= 31 + }) + + if (illegalCharsPattern.test(template) || hasControlChars) { + errors.push('路径模板包含非法字符') + } + + const openBraces = (template.match(/\{/g) || []).length + const closeBraces = (template.match(/\}/g) || []).length + if (openBraces !== closeBraces) { + errors.push('占位符括号未正确闭合') + } + + const knownPlaceholders = [ + 'Y', + 'y', + 'm', + 'd', + 'h', + 'i', + 's', + 'ms', + 'timestamp', + 'md5', + 'md5-16', + 'uuid', + 'filename', + 'ext', + ] + const placeholderPattern = /\{([^}]+)\}/g + let match: RegExpExecArray | null + placeholderPattern.lastIndex = 0 + + while ((match = placeholderPattern.exec(template)) !== null) { + const placeholder = match[1] + if ( + !knownPlaceholders.includes(placeholder) && + !/^str-\d+$/.test(placeholder) + ) { + errors.push(`未知的占位符: {${placeholder}}`) + } + } + + return { + valid: errors.length === 0, + errors, + } +} diff --git a/apps/core/src/utils/s3.util.ts b/apps/core/src/utils/s3.util.ts index c60ea52f11f..cb58f335dcd 100644 --- a/apps/core/src/utils/s3.util.ts +++ b/apps/core/src/utils/s3.util.ts @@ -161,7 +161,83 @@ export class S3Uploader { }) if (!response.ok) { - throw new Error(`Upload failed with status code: ${response.status}`) + const errorText = await response.text() + throw new Error( + `Upload failed with status code: ${response.status}, message: ${errorText}`, + ) + } + } + + async deleteFromS3(objectKey: string): Promise { + const service = 's3' + const date = new Date() + const xAmzDate = date.toISOString().replaceAll(/[:-]|\.\d{3}/g, '') + const dateStamp = xAmzDate.slice(0, 8) // YYYYMMDD + + const hashedPayload = crypto.createHash('sha256').update('').digest('hex') + + const url = new URL(this.endpoint) + const host = url.host + + const headers: Record = { + Host: host, + 'x-amz-date': xAmzDate, + 'x-amz-content-sha256': hashedPayload, + } + + const sortedHeaders = Object.keys(headers).sort() + const canonicalHeaders = sortedHeaders + .map((key) => `${key.toLowerCase()}:${headers[key].trim()}`) + .join('\n') + const signedHeaders = sortedHeaders + .map((key) => key.toLowerCase()) + .join(';') + + const canonicalRequest = [ + 'DELETE', + `/${this.bucket}/${objectKey}`, + '', + String(canonicalHeaders), + '', + signedHeaders, + hashedPayload, + ].join('\n') + + const algorithm = 'AWS4-HMAC-SHA256' + const credentialScope = `${dateStamp}/${this.region}/${service}/aws4_request` + const hashedCanonicalRequest = crypto + .createHash('sha256') + .update(canonicalRequest) + .digest('hex') + const stringToSign = [ + algorithm, + xAmzDate, + credentialScope, + hashedCanonicalRequest, + ].join('\n') + + const kSecret = Buffer.from(`AWS4${this.secretKey}`) + const kDate = this.hmacSha256(kSecret, dateStamp) + const kRegion = this.hmacSha256(kDate, this.region) + const kService = this.hmacSha256(kRegion, service) + const kSigning = this.hmacSha256(kService, 'aws4_request') + const signature = this.hmacSha256(kSigning, stringToSign).toString('hex') + const authorization = `${algorithm} Credential=${this.accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}` + const requestUrl = `${this.endpoint}/${this.bucket}/${objectKey}` + + const response = await fetch(requestUrl, { + method: 'DELETE', + headers: { + ...headers, + Authorization: authorization, + }, + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error( + `Delete failed with status code: ${response.status}, message: ${errorText}`, + ) } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a9e48d9d20..2e8d683afdb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -315,6 +315,9 @@ importers: ua-parser-js: specifier: 2.0.6 version: 2.0.6 + uuid: + specifier: ^13.0.0 + version: 13.0.0 vm2: specifier: 3.9.19 version: 3.9.19 @@ -6366,6 +6369,10 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + uuid@13.0.0: + resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==} + hasBin: true + uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -13360,6 +13367,8 @@ snapshots: utils-merge@1.0.1: {} + uuid@13.0.0: {} + uuid@8.3.2: {} v8-compile-cache-lib@3.0.1: {}