From e628fa6c63df1bb07355259f82c30f3f1d79d4c1 Mon Sep 17 00:00:00 2001 From: thanhtrieunguyen Date: Thu, 4 Jun 2026 16:46:56 +0700 Subject: [PATCH] feat: add SFTP and FTP backup destination support using rclone --- apps/dokploy/__test__/utils/backups.test.ts | 79 ++++++++++++++++++- .../settings/destination/constants.ts | 8 ++ .../destination/handle-destinations.tsx | 60 +++++++++++--- .../dokploy/server/api/routers/destination.ts | 29 ++++--- packages/server/src/utils/backups/compose.ts | 6 +- packages/server/src/utils/backups/index.ts | 17 ++-- packages/server/src/utils/backups/libsql.ts | 6 +- packages/server/src/utils/backups/mariadb.ts | 6 +- packages/server/src/utils/backups/mongo.ts | 6 +- packages/server/src/utils/backups/mysql.ts | 6 +- packages/server/src/utils/backups/postgres.ts | 6 +- packages/server/src/utils/backups/utils.ts | 42 +++++++++- .../server/src/utils/backups/web-server.ts | 23 ++++-- 13 files changed, 237 insertions(+), 57 deletions(-) diff --git a/apps/dokploy/__test__/utils/backups.test.ts b/apps/dokploy/__test__/utils/backups.test.ts index 2c1e5decc9..f5c859454b 100644 --- a/apps/dokploy/__test__/utils/backups.test.ts +++ b/apps/dokploy/__test__/utils/backups.test.ts @@ -1,5 +1,18 @@ -import { normalizeS3Path } from "@dokploy/server/utils/backups/utils"; -import { describe, expect, test } from "vitest"; +import { + getRclonePathAndFlags, + normalizeS3Path, +} from "@dokploy/server/utils/backups/utils"; +import { describe, expect, test, vi } from "vitest"; + +vi.mock("node:child_process", () => ({ + exec: (cmd: string, cb: any) => { + if (cmd.startsWith("rclone obscure")) { + cb(null, { stdout: "obscured_pass" }); + } else { + cb(null, { stdout: "" }); + } + }, +})); describe("normalizeS3Path", () => { test("should handle empty and whitespace-only prefix", () => { @@ -59,3 +72,65 @@ describe("normalizeS3Path", () => { expect(normalizeS3Path("instance-backups")).toBe("instance-backups/"); }); }); + +describe("getRclonePathAndFlags", () => { + test("should return correct flags and path for S3", async () => { + const destination = { + provider: "aws", + accessKey: "access", + secretAccessKey: "secret", + bucket: "mybucket", + region: "us-east-1", + endpoint: "https://s3.amazonaws.com", + }; + const { flags, path } = await getRclonePathAndFlags( + destination as any, + "mypath", + ); + expect(flags).toContain('--s3-access-key-id="access"'); + expect(flags).toContain('--s3-secret-access-key="secret"'); + expect(path).toBe(":s3:mybucket/mypath"); + }); + + test("should return correct on-the-fly connection string for SFTP", async () => { + const destination = { + provider: "sftp", + accessKey: "sftpuser", + secretAccessKey: "sftppass", + bucket: "sftppath", + region: "2022", + endpoint: "sftp.example.com", + }; + const { flags, path } = await getRclonePathAndFlags( + destination as any, + "mypath", + ); + expect(flags).toEqual([]); + expect(path).toContain( + ':sftp,host="sftp.example.com",port="2022",user="sftpuser"', + ); + expect(path).toContain('pass="obscured_pass"'); + expect(path.endsWith(":sftppath/mypath")).toBe(true); + }); + + test("should return correct on-the-fly connection string for FTP", async () => { + const destination = { + provider: "ftp", + accessKey: "ftpuser", + secretAccessKey: "ftppass", + bucket: "ftppath", + region: "21", + endpoint: "ftp.example.com", + }; + const { flags, path } = await getRclonePathAndFlags( + destination as any, + "mypath", + ); + expect(flags).toEqual([]); + expect(path).toContain( + ':ftp,host="ftp.example.com",port="21",user="ftpuser"', + ); + expect(path).toContain('pass="obscured_pass"'); + expect(path.endsWith(":ftppath/mypath")).toBe(true); + }); +}); diff --git a/apps/dokploy/components/dashboard/settings/destination/constants.ts b/apps/dokploy/components/dashboard/settings/destination/constants.ts index f43e47d1a1..c066451f69 100644 --- a/apps/dokploy/components/dashboard/settings/destination/constants.ts +++ b/apps/dokploy/components/dashboard/settings/destination/constants.ts @@ -130,4 +130,12 @@ export const S3_PROVIDERS: Array<{ key: "Other", name: "Any other S3 compatible provider", }, + { + key: "sftp", + name: "SFTP (SSH File Transfer Protocol)", + }, + { + key: "ftp", + name: "FTP (File Transfer Protocol)", + }, ]; diff --git a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx index e7ecf92b2b..444ff80659 100644 --- a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx +++ b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx @@ -113,6 +113,9 @@ export const HandleDestinations = ({ destinationId }: Props) => { name: "additionalFlags", }); + const currentProvider = form.watch("provider"); + const isSftpOrFtp = ["sftp", "ftp"].includes(currentProvider || ""); + useEffect(() => { if (destination) { form.reset({ @@ -196,7 +199,9 @@ export const HandleDestinations = ({ destinationId }: Props) => { const endpoint = form.getValues("endpoint"); const region = form.getValues("region"); - const connectionString = `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`; + const connectionString = isSftpOrFtp + ? `:${provider},host=${endpoint},port=${region || (provider === "sftp" ? "22" : "21")},user=${accessKey},pass=XXX:${bucket}` + : `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`; await testConnection({ provider, @@ -291,7 +296,7 @@ export const HandleDestinations = ({ destinationId }: Props) => { > - + @@ -318,9 +323,14 @@ export const HandleDestinations = ({ destinationId }: Props) => { render={({ field }) => { return ( - Access Key Id + + {isSftpOrFtp ? "Username" : "Access Key Id"} + - + @@ -333,10 +343,16 @@ export const HandleDestinations = ({ destinationId }: Props) => { render={({ field }) => (
- Secret Access Key + + {isSftpOrFtp ? "Password" : "Secret Access Key"} +
- + @@ -348,10 +364,17 @@ export const HandleDestinations = ({ destinationId }: Props) => { render={({ field }) => (
- Bucket + + {isSftpOrFtp ? "Path / Directory" : "Bucket"} +
- + @@ -363,10 +386,19 @@ export const HandleDestinations = ({ destinationId }: Props) => { render={({ field }) => (
- Region + {isSftpOrFtp ? "Port" : "Region"}
- + @@ -377,10 +409,14 @@ export const HandleDestinations = ({ destinationId }: Props) => { name="endpoint" render={({ field }) => ( - Endpoint + {isSftpOrFtp ? "Host" : "Endpoint"} diff --git a/apps/dokploy/server/api/routers/destination.ts b/apps/dokploy/server/api/routers/destination.ts index cf7395a3f7..ba8cb552ba 100644 --- a/apps/dokploy/server/api/routers/destination.ts +++ b/apps/dokploy/server/api/routers/destination.ts @@ -6,6 +6,7 @@ import { IS_CLOUD, removeDestinationById, updateDestinationById, + getRclonePathAndFlags, } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { TRPCError } from "@trpc/server"; @@ -57,25 +58,29 @@ export const destinationRouter = createTRPCRouter({ additionalFlags, } = input; try { - const rcloneFlags = [ - `--s3-access-key-id="${accessKey}"`, - `--s3-secret-access-key="${secretAccessKey}"`, - `--s3-region="${region}"`, - `--s3-endpoint="${endpoint}"`, - "--s3-no-check-bucket", - "--s3-force-path-style", + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags( + { + secretAccessKey, + bucket, + region, + endpoint, + accessKey, + provider, + } as any, + "", + ); + + rcloneFlags.push( "--retries 1", "--low-level-retries 1", "--timeout 10s", "--contimeout 5s", - ]; - if (provider) { - rcloneFlags.unshift(`--s3-provider="${provider}"`); - } + ); + if (additionalFlags?.length) { rcloneFlags.push(...additionalFlags); } - const rcloneDestination = `:s3:${bucket}`; const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} "${rcloneDestination}"`; if (IS_CLOUD && !input.serverId) { diff --git a/packages/server/src/utils/backups/compose.ts b/packages/server/src/utils/backups/compose.ts index 6640590b03..fe040d9d6f 100644 --- a/packages/server/src/utils/backups/compose.ts +++ b/packages/server/src/utils/backups/compose.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -34,8 +34,8 @@ export const runComposeBackup = async ( }); try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; const backupCommand = getBackupCommand( diff --git a/packages/server/src/utils/backups/index.ts b/packages/server/src/utils/backups/index.ts index 876579cb12..e6baa4a197 100644 --- a/packages/server/src/utils/backups/index.ts +++ b/packages/server/src/utils/backups/index.ts @@ -10,7 +10,11 @@ import { startLogCleanup } from "../access-log/handler"; import { cleanupAll } from "../docker/utils"; import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup"; import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils"; +import { + normalizeS3Path, + scheduleBackup, + getRclonePathAndFlags, +} from "./utils"; export const initCronJobs = async () => { console.log("Setting up cron jobs...."); @@ -131,17 +135,20 @@ export const keepLatestNBackups = async ( if (!backup.keepLatestCount) return; try { - const rcloneFlags = getS3Credentials(backup.destination); const appName = getServiceAppName(backup); - const backupFilesPath = `:s3:${backup.destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`; + const { flags: rcloneFlags, path: backupFilesPath } = + await getRclonePathAndFlags( + backup.destination, + `${appName}/${normalizeS3Path(backup.prefix)}`, + ); // --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone - const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`; + const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" "${backupFilesPath}"`; // when we pipe the above command with this one, we only get the list of files we want to delete const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`; // this command deletes the files // to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{} - const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`; + const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} "${backupFilesPath}{}"`; const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`; diff --git a/packages/server/src/utils/backups/libsql.ts b/packages/server/src/utils/backups/libsql.ts index a994db8bd6..cfc4265c64 100644 --- a/packages/server/src/utils/backups/libsql.ts +++ b/packages/server/src/utils/backups/libsql.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -33,8 +33,8 @@ export const runLibsqlBackup = async ( const backupFileName = `${getBackupTimestamp()}.sql.gz`; const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; diff --git a/packages/server/src/utils/backups/mariadb.ts b/packages/server/src/utils/backups/mariadb.ts index dea22ff189..bec302547d 100644 --- a/packages/server/src/utils/backups/mariadb.ts +++ b/packages/server/src/utils/backups/mariadb.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -32,8 +32,8 @@ export const runMariadbBackup = async ( description: "MariaDB Backup", }); try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; const backupCommand = getBackupCommand( diff --git a/packages/server/src/utils/backups/mongo.ts b/packages/server/src/utils/backups/mongo.ts index cebece14f7..1275c26a02 100644 --- a/packages/server/src/utils/backups/mongo.ts +++ b/packages/server/src/utils/backups/mongo.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -29,8 +29,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { description: "MongoDB Backup", }); try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; const backupCommand = getBackupCommand( diff --git a/packages/server/src/utils/backups/mysql.ts b/packages/server/src/utils/backups/mysql.ts index a72f598806..22348d1afd 100644 --- a/packages/server/src/utils/backups/mysql.ts +++ b/packages/server/src/utils/backups/mysql.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -30,8 +30,8 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { }); try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; diff --git a/packages/server/src/utils/backups/postgres.ts b/packages/server/src/utils/backups/postgres.ts index 30a88db2b3..f26ba33f90 100644 --- a/packages/server/src/utils/backups/postgres.ts +++ b/packages/server/src/utils/backups/postgres.ts @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync"; import { getBackupCommand, getBackupTimestamp, - getS3Credentials, + getRclonePathAndFlags, normalizeS3Path, } from "./utils"; @@ -33,8 +33,8 @@ export const runPostgresBackup = async ( const backupFileName = `${getBackupTimestamp()}.sql.gz`; const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`; try { - const rcloneFlags = getS3Credentials(destination); - const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; + const { flags: rcloneFlags, path: rcloneDestination } = + await getRclonePathAndFlags(destination, bucketDestination); const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index 365ebff415..5420322cc8 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -2,6 +2,9 @@ import { logger } from "@dokploy/server/lib/logger"; import type { BackupSchedule } from "@dokploy/server/services/backup"; import type { Destination } from "@dokploy/server/services/destination"; import { scheduledJobs, scheduleJob } from "node-schedule"; +import { exec } from "node:child_process"; +import { promisify } from "node:util"; +const execPromise = promisify(exec); import { keepLatestNBackups } from "."; import { runComposeBackup } from "./compose"; import { runLibsqlBackup } from "./libsql"; @@ -257,6 +260,11 @@ export const getBackupCommand = ( ) => { const containerSearch = getContainerSearchCommand(backup); const backupCommand = generateBackupCommand(backup); + const destinationType = ["sftp", "ftp"].includes( + backup.destination?.provider || "", + ) + ? backup.destination?.provider?.toUpperCase() || "remote" + : "S3"; logger.info( { @@ -289,16 +297,44 @@ export const getBackupCommand = ( } echo "[$(date)] ✅ backup completed successfully" >> ${logPath}; - echo "[$(date)] Starting upload to S3..." >> ${logPath}; + echo "[$(date)] Starting upload to ${destinationType}..." >> ${logPath}; # Run the upload command and capture the exit status UPLOAD_OUTPUT=$(${backupCommand} | ${rcloneCommand} 2>&1 >/dev/null) || { - echo "[$(date)] ❌ Error: Upload to S3 failed" >> ${logPath}; + echo "[$(date)] ❌ Error: Upload to ${destinationType} failed" >> ${logPath}; echo "Error: $UPLOAD_OUTPUT" >> ${logPath}; exit 1; } - echo "[$(date)] ✅ Upload to S3 completed successfully" >> ${logPath}; + echo "[$(date)] ✅ Upload to ${destinationType} completed successfully" >> ${logPath}; echo "Backup done ✅" >> ${logPath}; `; }; + +export const obscurePassword = async (password: string) => { + try { + const { stdout } = await execPromise( + `rclone obscure "${password.replace(/"/g, '\\"')}"`, + ); + return stdout.trim(); + } catch (error) { + logger.error("Error obscuring password with rclone", error); + return password; + } +}; + +export const getRclonePathAndFlags = async ( + destination: Destination, + subPath: string, +) => { + const isS3 = !["sftp", "ftp"].includes(destination.provider || ""); + if (isS3) { + const flags = getS3Credentials(destination); + const path = `:s3:${destination.bucket}/${subPath}`; + return { flags, path }; + } + const provider = destination.provider; + const obscuredPass = await obscurePassword(destination.secretAccessKey); + const path = `:${provider},host="${destination.endpoint}",port="${destination.region}",user="${destination.accessKey}",pass="${obscuredPass}":${destination.bucket}/${subPath}`; + return { flags: [], path }; +}; diff --git a/packages/server/src/utils/backups/web-server.ts b/packages/server/src/utils/backups/web-server.ts index 712cc08090..ceaaa8c071 100644 --- a/packages/server/src/utils/backups/web-server.ts +++ b/packages/server/src/utils/backups/web-server.ts @@ -11,7 +11,11 @@ import { import { findDestinationById } from "@dokploy/server/services/destination"; import { sendDokployBackupNotifications } from "../notifications/dokploy-backup"; import { execAsync } from "../process/execAsync"; -import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; +import { + getBackupTimestamp, + getRclonePathAndFlags, + normalizeS3Path, +} from "./utils"; function formatBytes(bytes?: number) { if (bytes === undefined) return "Unknown size"; @@ -36,12 +40,14 @@ export const runWebServerBackup = async (backup: BackupSchedule) => { let computedBackupSize: number | undefined; try { const destination = await findDestinationById(backup.destinationId); - const rcloneFlags = getS3Credentials(destination); const timestamp = getBackupTimestamp(); const { BASE_PATH } = paths(); const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-")); const backupFileName = `webserver-backup-${timestamp}.zip`; - const s3Path = `:s3:${destination.bucket}/${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`; + const { flags: rcloneFlags, path: s3Path } = await getRclonePathAndFlags( + destination, + `${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`, + ); try { await execAsync(`mkdir -p ${tempDir}/filesystem`); @@ -99,9 +105,16 @@ export const runWebServerBackup = async (backup: BackupSchedule) => { } const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${zipPath}" "${s3Path}"`; - writeStream.write("Running command to upload backup to S3\n"); + const destinationType = ["sftp", "ftp"].includes( + destination.provider || "", + ) + ? destination.provider?.toUpperCase() || "remote" + : "S3"; + writeStream.write( + `Running command to upload backup to ${destinationType}\n`, + ); await execAsync(uploadCommand); - writeStream.write("Uploaded backup to S3 ✅\n"); + writeStream.write(`Uploaded backup to ${destinationType} ✅\n`); writeStream.end(); await sendDokployBackupNotifications({ type: "success",