Add an rclone based option to populate the R2 cache

.changeset/recovered-rclone-batch-upload.md

@@ -0,0 +1,38 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: optional batch upload via `rclone` for fast R2 cache population.
+
+**Key Changes:**
+
+1. **Optional `rclone` Upload**: Configure R2 credentials to opt in to `rclone` based batch uploads:
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `CF_ACCOUNT_ID`
+
+2. **Automatic Detection**: When credentials are detected and the target is `remote`, `rclone` batch upload is automatically used. Local targets always use `wrangler r2 bulk put` directly.
+
+3. **Smart Fallback**: If credentials are not configured or if `rclone` fails, the CLI falls back to `wrangler r2 bulk put`.
+
+**Benefits (when batch upload is enabled):**
+
+`rclone` uses the S3 API which is not subject to the REST API limits
+
+**Usage:**
+
+Add the secrets in a `.env`/`.dev.vars` file in your project root,
+
+```bash
+R2_ACCESS_KEY_ID=your_key
+R2_SECRET_ACCESS_KEY=your_secret
+CF_ACCOUNT_ID=your_account
+```
+
+You can also set the environment variables for CI builds.
+
+**Notes:**
+
+- You can follow documentation <https://developers.cloudflare.com/r2/api/tokens/> for creating API tokens with appropriate permissions for R2 access.
+- `rclone` may not be supported on all platforms.

packages/cloudflare/package.json

@@ -58,6 +58,7 @@
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
+		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},
@@ -68,6 +69,7 @@
 		"@types/mock-fs": "catalog:",
 		"@types/node": "catalog:",
 		"@types/picomatch": "^4.0.0",
+		"@types/rclone.js": "^0.6.1",
 		"@types/yargs": "catalog:",
 		"diff": "^8.0.2",
 		"esbuild": "catalog:",

packages/cloudflare/src/api/cloudflare-context.ts

@@ -83,8 +83,16 @@ declare global {
 		CF_PREVIEW_DOMAIN?: string;
 		// Should have the `Workers Scripts:Read` permission
 		CF_WORKERS_SCRIPTS_API_TOKEN?: string;
-		// Cloudflare account id - needed for skew protection
+
+		// Cloudflare account id - needed for skew protection and R2 batch population
 		CF_ACCOUNT_ID?: string;
+
+		// R2 S3 API credentials used by `rclone`
+		// NOTES:
+		// - `rclone` may not be supported on all platforms
+		// - `rclone` is not supported in development
+		R2_ACCESS_KEY_ID?: string;
+		R2_SECRET_ACCESS_KEY?: string;
 	}
 }
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -2,9 +2,14 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
 import mockFs from "mock-fs";
+import rclone from "rclone.js";
 import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
+import type { Unstable_Config as WranglerConfig } from "wrangler";
 
+import { runWrangler } from "../utils/run-wrangler.js";
+import type { WorkerEnvVar } from "./helpers.js";
 import { getCacheAssets, populateCache } from "./populate-cache.js";
 
 describe("getCacheAssets", () => {
@@ -78,102 +83,120 @@ vi.mock("./helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
+// Mock `rclone.js` promises API to simulate successful copy operations by default
+vi.mock("rclone.js", () => ({
+	default: {
+		promises: {
+			copy: vi.fn(() => Promise.resolve("")),
+		},
+	},
+}));
+
 describe("populateCache", () => {
-	const setupMockFileSystem = () => {
-		mockFs({
-			"/test/output": {
-				cache: {
-					buildID: {
-						path: {
-							to: {
-								"test.cache": JSON.stringify({ data: "test" }),
+	describe("R2 incremental cache", () => {
+		const buildOptions = { outputDir: "/test/output" } as BuildOptions;
+
+		const openNextConfig = {
+			default: {
+				override: {
+					incrementalCache: () => Promise.resolve({ name: "cf-r2-incremental-cache" }),
+				},
+			},
+		} as OpenNextConfig;
+
+		const wranglerConfig = {
+			r2_buckets: [
+				{
+					binding: "NEXT_INC_CACHE_R2_BUCKET",
+					bucket_name: "test-bucket",
+				},
+			],
+		} as WranglerConfig;
+
+		const r2Credentials = {
+			R2_ACCESS_KEY_ID: "test_access_key",
+			R2_SECRET_ACCESS_KEY: "test_secret_key",
+			CF_ACCOUNT_ID: "test_account_id",
+		} as WorkerEnvVar;
+
+		const setupMockFileSystem = () => {
+			mockFs({
+				"/test/output": {
+					cache: {
+						buildID: {
+							path: {
+								to: {
+									"test.cache": JSON.stringify({ data: "test" }),
+								},
 							},
 						},
 					},
 				},
-			},
-		});
-	};
-
-	describe.each([{ target: "local" as const }, { target: "remote" as const }])(
-		"R2 incremental cache",
-		({ target }) => {
-			afterEach(() => {
-				mockFs.restore();
 			});
+		};
 
-			test(target, async () => {
-				const { runWrangler } = await import("../utils/run-wrangler.js");
+		afterEach(() => {
+			mockFs.restore();
+			vi.unstubAllEnvs();
+		});
 
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
+		test("uses `wrangler r2 bulk put` for local target", async () => {
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+			vi.mocked(rclone.promises.copy).mockClear();
+
+			await populateCache(
+				buildOptions,
+				openNextConfig,
+				wranglerConfig,
+				{ target: "local" as const, shouldUsePreviewId: false },
+				r2Credentials
+			);
+
+			expect(runWrangler).toHaveBeenCalled();
+			expect(rclone.promises.copy).not.toHaveBeenCalled();
+		});
 
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-				);
-
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
-					expect.objectContaining({ target })
-				);
-			});
+		test("uses `rclone` for remote target when R2 credentials are provided", async () => {
+			setupMockFileSystem();
+			vi.mocked(rclone.promises.copy).mockClear();
+
+			await populateCache(
+				buildOptions,
+				openNextConfig,
+				wranglerConfig,
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				r2Credentials
+			);
+
+			expect(rclone.promises.copy).toHaveBeenCalledWith(
+				expect.any(String), // staging directory
+				"r2:test-bucket",
+				expect.objectContaining({
+					progress: true,
+					transfers: expect.any(Number),
+					checkers: expect.any(Number),
+					env: expect.objectContaining({
+						RCLONE_CONFIG: expect.any(String), // `rclone` config content with R2 credentials
+					}),
+				})
+			);
+		});
 
-			test(`${target} using jurisdiction`, async () => {
-				const { runWrangler } = await import("../utils/run-wrangler.js");
+		test("fallback to `wrangler r2 bulk put` when `rclone` fails", async () => {
+			setupMockFileSystem();
+			vi.mocked(rclone.promises.copy).mockRejectedValueOnce(new Error("rclone copy failed with exit code 7"));
+			vi.mocked(runWrangler).mockClear();
 
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
+			await populateCache(
+				buildOptions,
+				openNextConfig,
+				wranglerConfig,
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				r2Credentials
+			);
 
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-								jurisdiction: "eu",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-				);
-
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]),
-					expect.objectContaining({ target })
-				);
-			});
-		}
-	);
+			expect(runWrangler).toHaveBeenCalled();
+		});
+	});
 });