Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 77 additions & 2 deletions apps/dokploy/__test__/utils/backups.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,18 @@
import { normalizeS3Path } from "@dokploy/server/utils/backups/utils";
import { describe, expect, test } from "vitest";
import {
getRclonePathAndFlags,
normalizeS3Path,
} from "@dokploy/server/utils/backups/utils";
import { describe, expect, test, vi } from "vitest";

vi.mock("node:child_process", () => ({
exec: (cmd: string, cb: any) => {
if (cmd.startsWith("rclone obscure")) {
cb(null, { stdout: "obscured_pass" });
} else {
cb(null, { stdout: "" });
}
},
}));

describe("normalizeS3Path", () => {
test("should handle empty and whitespace-only prefix", () => {
Expand Down Expand Up @@ -59,3 +72,65 @@ describe("normalizeS3Path", () => {
expect(normalizeS3Path("instance-backups")).toBe("instance-backups/");
});
});

describe("getRclonePathAndFlags", () => {
test("should return correct flags and path for S3", async () => {
const destination = {
provider: "aws",
accessKey: "access",
secretAccessKey: "secret",
bucket: "mybucket",
region: "us-east-1",
endpoint: "https://s3.amazonaws.com",
};
const { flags, path } = await getRclonePathAndFlags(
destination as any,
"mypath",
);
expect(flags).toContain('--s3-access-key-id="access"');
expect(flags).toContain('--s3-secret-access-key="secret"');
expect(path).toBe(":s3:mybucket/mypath");
});

test("should return correct on-the-fly connection string for SFTP", async () => {
const destination = {
provider: "sftp",
accessKey: "sftpuser",
secretAccessKey: "sftppass",
bucket: "sftppath",
region: "2022",
endpoint: "sftp.example.com",
};
const { flags, path } = await getRclonePathAndFlags(
destination as any,
"mypath",
);
expect(flags).toEqual([]);
expect(path).toContain(
':sftp,host="sftp.example.com",port="2022",user="sftpuser"',
);
expect(path).toContain('pass="obscured_pass"');
expect(path.endsWith(":sftppath/mypath")).toBe(true);
});

test("should return correct on-the-fly connection string for FTP", async () => {
const destination = {
provider: "ftp",
accessKey: "ftpuser",
secretAccessKey: "ftppass",
bucket: "ftppath",
region: "21",
endpoint: "ftp.example.com",
};
const { flags, path } = await getRclonePathAndFlags(
destination as any,
"mypath",
);
expect(flags).toEqual([]);
expect(path).toContain(
':ftp,host="ftp.example.com",port="21",user="ftpuser"',
);
expect(path).toContain('pass="obscured_pass"');
expect(path.endsWith(":ftppath/mypath")).toBe(true);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -130,4 +130,12 @@ export const S3_PROVIDERS: Array<{
key: "Other",
name: "Any other S3 compatible provider",
},
{
key: "sftp",
name: "SFTP (SSH File Transfer Protocol)",
},
{
key: "ftp",
name: "FTP (File Transfer Protocol)",
},
];
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ export const HandleDestinations = ({ destinationId }: Props) => {
name: "additionalFlags",
});

const currentProvider = form.watch("provider");
const isSftpOrFtp = ["sftp", "ftp"].includes(currentProvider || "");

useEffect(() => {
if (destination) {
form.reset({
Expand Down Expand Up @@ -196,7 +199,9 @@ export const HandleDestinations = ({ destinationId }: Props) => {
const endpoint = form.getValues("endpoint");
const region = form.getValues("region");

const connectionString = `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`;
const connectionString = isSftpOrFtp
? `:${provider},host=${endpoint},port=${region || (provider === "sftp" ? "22" : "21")},user=${accessKey},pass=XXX:${bucket}`
: `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`;

await testConnection({
provider,
Expand Down Expand Up @@ -291,7 +296,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a S3 Provider" />
<SelectValue placeholder="Select a Provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
Expand All @@ -318,9 +323,14 @@ export const HandleDestinations = ({ destinationId }: Props) => {
render={({ field }) => {
return (
<FormItem>
<FormLabel>Access Key Id</FormLabel>
<FormLabel>
{isSftpOrFtp ? "Username" : "Access Key Id"}
</FormLabel>
<FormControl>
<Input placeholder={"xcas41dasde"} {...field} />
<Input
placeholder={isSftpOrFtp ? "username" : "xcas41dasde"}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
Expand All @@ -333,10 +343,16 @@ export const HandleDestinations = ({ destinationId }: Props) => {
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Secret Access Key</FormLabel>
<FormLabel>
{isSftpOrFtp ? "Password" : "Secret Access Key"}
</FormLabel>
</div>
<FormControl>
<Input placeholder={"asd123asdasw"} {...field} />
<Input
type={isSftpOrFtp ? "password" : "text"}
placeholder={isSftpOrFtp ? "password" : "asd123asdasw"}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
Expand All @@ -348,10 +364,17 @@ export const HandleDestinations = ({ destinationId }: Props) => {
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Bucket</FormLabel>
<FormLabel>
{isSftpOrFtp ? "Path / Directory" : "Bucket"}
</FormLabel>
</div>
<FormControl>
<Input placeholder={"dokploy-bucket"} {...field} />
<Input
placeholder={
isSftpOrFtp ? "/backups/dokploy" : "dokploy-bucket"
}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
Expand All @@ -363,10 +386,19 @@ export const HandleDestinations = ({ destinationId }: Props) => {
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Region</FormLabel>
<FormLabel>{isSftpOrFtp ? "Port" : "Region"}</FormLabel>
</div>
<FormControl>
<Input placeholder={"us-east-1"} {...field} />
<Input
placeholder={
isSftpOrFtp
? currentProvider === "sftp"
? "22"
: "21"
: "us-east-1"
}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
Expand All @@ -377,10 +409,14 @@ export const HandleDestinations = ({ destinationId }: Props) => {
name="endpoint"
render={({ field }) => (
<FormItem>
<FormLabel>Endpoint</FormLabel>
<FormLabel>{isSftpOrFtp ? "Host" : "Endpoint"}</FormLabel>
<FormControl>
<Input
placeholder={"https://us.bucket.aws/s3"}
placeholder={
isSftpOrFtp
? "sftp.example.com"
: "https://us.bucket.aws/s3"
}
{...field}
/>
</FormControl>
Expand Down
29 changes: 17 additions & 12 deletions apps/dokploy/server/api/routers/destination.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
IS_CLOUD,
removeDestinationById,
updateDestinationById,
getRclonePathAndFlags,
} from "@dokploy/server";
import { db } from "@dokploy/server/db";
import { TRPCError } from "@trpc/server";
Expand Down Expand Up @@ -57,25 +58,29 @@ export const destinationRouter = createTRPCRouter({
additionalFlags,
} = input;
try {
const rcloneFlags = [
`--s3-access-key-id="${accessKey}"`,
`--s3-secret-access-key="${secretAccessKey}"`,
`--s3-region="${region}"`,
`--s3-endpoint="${endpoint}"`,
"--s3-no-check-bucket",
"--s3-force-path-style",
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(
{
secretAccessKey,
bucket,
region,
endpoint,
accessKey,
provider,
} as any,
"",
);

rcloneFlags.push(
"--retries 1",
"--low-level-retries 1",
"--timeout 10s",
"--contimeout 5s",
];
if (provider) {
rcloneFlags.unshift(`--s3-provider="${provider}"`);
}
);

if (additionalFlags?.length) {
rcloneFlags.push(...additionalFlags);
}
const rcloneDestination = `:s3:${bucket}`;
const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

if (IS_CLOUD && !input.serverId) {
Expand Down
6 changes: 3 additions & 3 deletions packages/server/src/utils/backups/compose.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";

Expand All @@ -34,8 +34,8 @@ export const runComposeBackup = async (
});

try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

const backupCommand = getBackupCommand(
Expand Down
17 changes: 12 additions & 5 deletions packages/server/src/utils/backups/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ import { startLogCleanup } from "../access-log/handler";
import { cleanupAll } from "../docker/utils";
import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils";
import {
normalizeS3Path,
scheduleBackup,
getRclonePathAndFlags,
} from "./utils";

export const initCronJobs = async () => {
console.log("Setting up cron jobs....");
Expand Down Expand Up @@ -131,17 +135,20 @@ export const keepLatestNBackups = async (
if (!backup.keepLatestCount) return;

try {
const rcloneFlags = getS3Credentials(backup.destination);
const appName = getServiceAppName(backup);
const backupFilesPath = `:s3:${backup.destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`;
const { flags: rcloneFlags, path: backupFilesPath } =
await getRclonePathAndFlags(
backup.destination,
`${appName}/${normalizeS3Path(backup.prefix)}`,
);

// --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`;
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" "${backupFilesPath}"`;
// when we pipe the above command with this one, we only get the list of files we want to delete
const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
// this command deletes the files
// to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{}
const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`;
const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} "${backupFilesPath}{}"`;

const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;

Expand Down
6 changes: 3 additions & 3 deletions packages/server/src/utils/backups/libsql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";

Expand All @@ -33,8 +33,8 @@ export const runLibsqlBackup = async (
const backupFileName = `${getBackupTimestamp()}.sql.gz`;
const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);

const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

Expand Down
6 changes: 3 additions & 3 deletions packages/server/src/utils/backups/mariadb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";

Expand All @@ -32,8 +32,8 @@ export const runMariadbBackup = async (
description: "MariaDB Backup",
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

const backupCommand = getBackupCommand(
Expand Down
6 changes: 3 additions & 3 deletions packages/server/src/utils/backups/mongo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { execAsync, execAsyncRemote } from "../process/execAsync";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";

Expand All @@ -29,8 +29,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
description: "MongoDB Backup",
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

const backupCommand = getBackupCommand(
Expand Down
Loading