Skip to content

Commit feac4be

Browse files
matt-aitkenmatt-aitken
committed
Added container tests (using Minio)
1 parent baa015c commit feac4be

File tree

4 files changed

+457
-0
lines changed

4 files changed

+457
-0
lines changed

‎apps/webapp/test/objectStore.test.ts‎

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
import { minioTest } from "@internal/testcontainers";
2+
import { type IOPacket } from "@trigger.dev/core/v3";
3+
import { PrismaClient } from "@trigger.dev/database";
4+
import { afterAll, describe, expect, it, vi } from "vitest";
5+
import {
6+
downloadPacketFromObjectStore,
7+
formatStorageUri,
8+
parseStorageUri,
9+
uploadPacketToObjectStore,
10+
} from "~/v3/objectStore.server";
11+
12+
// Extend the timeout for container tests
13+
vi.setConfig({ testTimeout: 30_000 });
14+
15+
// Helper to create a test environment
16+
async function createTestEnvironment(prisma: PrismaClient) {
17+
const suffix = Date.now().toString(36);
18+
19+
const org = await prisma.organization.create({
20+
data: {
21+
title: `Test Org ${suffix}`,
22+
slug: `test-org-${suffix}`,
23+
},
24+
});
25+
26+
const project = await prisma.project.create({
27+
data: {
28+
name: `Test Project ${suffix}`,
29+
slug: `test-project-${suffix}`,
30+
externalRef: `proj_test${suffix}`,
31+
organizationId: org.id,
32+
},
33+
});
34+
35+
const environment = await prisma.runtimeEnvironment.create({
36+
data: {
37+
slug: "dev",
38+
type: "DEVELOPMENT",
39+
organizationId: org.id,
40+
projectId: project.id,
41+
apiKey: `test_key_${suffix}`,
42+
pkApiKey: `test_pk_key_${suffix}`,
43+
shortcode: suffix.slice(0, 4),
44+
},
45+
include: {
46+
project: true,
47+
organization: true,
48+
},
49+
});
50+
51+
return environment;
52+
}
53+
54+
// Mock env module for testing
55+
const originalEnv = process.env;
56+
57+
// Create extended test with Prisma client
58+
const minioWithPrismaTest = minioTest.extend<{ prisma: PrismaClient }>({
59+
prisma: async ({}, use) => {
60+
const prisma = new PrismaClient({
61+
datasources: {
62+
db: {
63+
url: process.env.DATABASE_URL,
64+
},
65+
},
66+
});
67+
await use(prisma);
68+
await prisma.$disconnect();
69+
},
70+
});
71+
72+
describe("Object Storage", () => {
73+
describe("URI parsing functions", () => {
74+
it("should parse URI with protocol", () => {
75+
const result = parseStorageUri("s3://run_abc123/payload.json");
76+
expect(result).toEqual({
77+
protocol: "s3",
78+
path: "run_abc123/payload.json",
79+
});
80+
});
81+
82+
it("should parse URI with R2 protocol", () => {
83+
const result = parseStorageUri("r2://batch_xyz/item_0/payload.json");
84+
expect(result).toEqual({
85+
protocol: "r2",
86+
path: "batch_xyz/item_0/payload.json",
87+
});
88+
});
89+
90+
it("should parse legacy URI without protocol", () => {
91+
const result = parseStorageUri("run_abc123/payload.json");
92+
expect(result).toEqual({
93+
protocol: undefined,
94+
path: "run_abc123/payload.json",
95+
});
96+
});
97+
98+
it("should format URI with protocol", () => {
99+
const result = formatStorageUri("run_abc123/payload.json", "s3");
100+
expect(result).toBe("s3://run_abc123/payload.json");
101+
});
102+
103+
it("should format URI without protocol", () => {
104+
const result = formatStorageUri("run_abc123/payload.json");
105+
expect(result).toBe("run_abc123/payload.json");
106+
});
107+
});
108+
109+
minioWithPrismaTest(
110+
"should upload and download data without protocol (legacy)",
111+
async ({ minioConfig, prisma }) => {
112+
// Set up env for default provider
113+
process.env.OBJECT_STORE_BASE_URL = minioConfig.baseUrl;
114+
process.env.OBJECT_STORE_ACCESS_KEY_ID = minioConfig.accessKeyId;
115+
process.env.OBJECT_STORE_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
116+
process.env.OBJECT_STORE_REGION = minioConfig.region;
117+
process.env.OBJECT_STORE_SERVICE = "s3";
118+
delete process.env.OBJECT_STORE_DEFAULT_PROTOCOL;
119+
120+
const environment = await createTestEnvironment(prisma);
121+
122+
const testData = JSON.stringify({ test: "data", value: 123 });
123+
const filename = "test_run/payload.json";
124+
125+
// Upload
126+
const uploadedFilename = await uploadPacketToObjectStore(
127+
filename,
128+
testData,
129+
"application/json",
130+
environment as any
131+
);
132+
133+
// Should return filename without protocol (legacy)
134+
expect(uploadedFilename).toBe(filename);
135+
136+
// Download
137+
const packet: IOPacket = {
138+
data: uploadedFilename,
139+
dataType: "application/store",
140+
};
141+
142+
const downloadedPacket = await downloadPacketFromObjectStore(packet, environment as any);
143+
144+
expect(downloadedPacket.dataType).toBe("application/json");
145+
expect(downloadedPacket.data).toBe(testData);
146+
147+
// Cleanup
148+
await prisma.runtimeEnvironment.delete({ where: { id: environment.id } });
149+
await prisma.project.delete({ where: { id: environment.projectId } });
150+
await prisma.organization.delete({ where: { id: environment.organizationId } });
151+
}
152+
);
153+
154+
minioWithPrismaTest(
155+
"should upload and download data with protocol prefix",
156+
async ({ minioConfig, prisma }) => {
157+
// Set up env for named provider
158+
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl;
159+
process.env.OBJECT_STORE_S3_ACCESS_KEY_ID = minioConfig.accessKeyId;
160+
process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
161+
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
162+
process.env.OBJECT_STORE_S3_SERVICE = "s3";
163+
process.env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
164+
165+
const environment = await createTestEnvironment(prisma);
166+
167+
const testData = JSON.stringify({ test: "protocol-data", value: 456 });
168+
const filename = "test_run2/payload.json";
169+
170+
// Upload with protocol
171+
const uploadedFilename = await uploadPacketToObjectStore(
172+
filename,
173+
testData,
174+
"application/json",
175+
environment as any,
176+
"s3"
177+
);
178+
179+
// Should return filename with s3:// protocol
180+
expect(uploadedFilename).toBe("s3://test_run2/payload.json");
181+
182+
// Download
183+
const packet: IOPacket = {
184+
data: uploadedFilename,
185+
dataType: "application/store",
186+
};
187+
188+
const downloadedPacket = await downloadPacketFromObjectStore(packet, environment as any);
189+
190+
expect(downloadedPacket.dataType).toBe("application/json");
191+
expect(downloadedPacket.data).toBe(testData);
192+
193+
// Cleanup
194+
await prisma.runtimeEnvironment.delete({ where: { id: environment.id } });
195+
await prisma.project.delete({ where: { id: environment.projectId } });
196+
await prisma.organization.delete({ where: { id: environment.organizationId } });
197+
}
198+
);
199+
200+
minioWithPrismaTest(
201+
"should support migration from default provider to named provider",
202+
async ({ minioConfig, prisma }) => {
203+
const environment = await createTestEnvironment(prisma);
204+
205+
// Step 1: Upload old data without protocol (using default provider)
206+
process.env.OBJECT_STORE_BASE_URL = minioConfig.baseUrl;
207+
process.env.OBJECT_STORE_ACCESS_KEY_ID = minioConfig.accessKeyId;
208+
process.env.OBJECT_STORE_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
209+
process.env.OBJECT_STORE_REGION = minioConfig.region;
210+
process.env.OBJECT_STORE_SERVICE = "s3";
211+
delete process.env.OBJECT_STORE_DEFAULT_PROTOCOL;
212+
213+
const oldData = JSON.stringify({ legacy: true });
214+
const oldFilename = "old_run/payload.json";
215+
216+
const uploadedOldFilename = await uploadPacketToObjectStore(
217+
oldFilename,
218+
oldData,
219+
"application/json",
220+
environment as any
221+
);
222+
223+
expect(uploadedOldFilename).toBe(oldFilename); // No protocol
224+
225+
// Step 2: Configure new provider (S3) and set default protocol
226+
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl; // Same MinIO for testing
227+
process.env.OBJECT_STORE_S3_ACCESS_KEY_ID = minioConfig.accessKeyId;
228+
process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
229+
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
230+
process.env.OBJECT_STORE_S3_SERVICE = "s3";
231+
process.env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
232+
233+
// Step 3: Upload new data with protocol
234+
const newData = JSON.stringify({ new: true });
235+
const newFilename = "new_run/payload.json";
236+
237+
const uploadedNewFilename = await uploadPacketToObjectStore(
238+
newFilename,
239+
newData,
240+
"application/json",
241+
environment as any,
242+
"s3"
243+
);
244+
245+
expect(uploadedNewFilename).toBe("s3://new_run/payload.json"); // Has protocol
246+
247+
// Step 4: Verify both can be downloaded
248+
// Old data (no protocol, uses default provider)
249+
const oldPacket: IOPacket = {
250+
data: uploadedOldFilename,
251+
dataType: "application/store",
252+
};
253+
const downloadedOld = await downloadPacketFromObjectStore(oldPacket, environment as any);
254+
expect(downloadedOld.data).toBe(oldData);
255+
256+
// New data (with protocol, uses named provider)
257+
const newPacket: IOPacket = {
258+
data: uploadedNewFilename,
259+
dataType: "application/store",
260+
};
261+
const downloadedNew = await downloadPacketFromObjectStore(newPacket, environment as any);
262+
expect(downloadedNew.data).toBe(newData);
263+
264+
// Cleanup
265+
await prisma.runtimeEnvironment.delete({ where: { id: environment.id } });
266+
await prisma.project.delete({ where: { id: environment.projectId } });
267+
await prisma.organization.delete({ where: { id: environment.organizationId } });
268+
}
269+
);
270+
271+
// Cleanup env after all tests
272+
afterAll(() => {
273+
process.env = originalEnv;
274+
});
275+
});

‎internal-packages/testcontainers/src/index.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,18 @@ import {
99
createElectricContainer,
1010
createPostgresContainer,
1111
createRedisContainer,
12+
createMinIOContainer,
1213
useContainer,
1314
withContainerSetup,
1415
} from "./utils";
1516
import { getTaskMetadata, logCleanup, logSetup } from "./logs";
1617
import { StartedClickHouseContainer } from "./clickhouse";
18+
import { StartedMinIOContainer, type MinIOConnectionConfig } from "./minio";
1719
import { ClickHouseClient, createClient } from "@clickhouse/client";
1820

1921
export { assertNonNullable } from "./utils";
2022
export { logCleanup };
23+
export type { MinIOConnectionConfig };
2124

2225
type NetworkContext = { network: StartedNetwork };
2326

@@ -40,11 +43,17 @@ export type PostgresAndRedisContext = NetworkContext & PostgresContext & RedisCo
4043
export type ContainerWithElectricAndRedisContext = ContainerContext & ElectricContext;
4144
export type ContainerWithElectricContext = NetworkContext & PostgresContext & ElectricContext;
4245

46+
type MinIOContext = NetworkContext & {
47+
minioContainer: StartedMinIOContainer;
48+
minioConfig: MinIOConnectionConfig;
49+
};
50+
4351
export type {
4452
StartedNetwork,
4553
StartedPostgreSqlContainer,
4654
StartedRedisContainer,
4755
StartedClickHouseContainer,
56+
StartedMinIOContainer,
4857
};
4958

5059
type Use<T> = (value: T) => Promise<void>;
@@ -257,3 +266,29 @@ export const containerWithElectricAndRedisTest = test.extend<ContainerWithElectr
257266
clickhouseContainer,
258267
clickhouseClient,
259268
});
269+
270+
const minioContainer = async (
271+
{ network, task }: { network: StartedNetwork } & TaskContext,
272+
use: Use<StartedMinIOContainer>
273+
) => {
274+
const { container, metadata } = await withContainerSetup({
275+
name: "minioContainer",
276+
task,
277+
setup: createMinIOContainer(network),
278+
});
279+
280+
await useContainer("minioContainer", { container, task, use: () => use(container) });
281+
};
282+
283+
const minioConfig = async (
284+
{ minioContainer }: { minioContainer: StartedMinIOContainer },
285+
use: Use<MinIOConnectionConfig>
286+
) => {
287+
await use(minioContainer.getConnectionConfig());
288+
};
289+
290+
export const minioTest = test.extend<MinIOContext>({
291+
network,
292+
minioContainer,
293+
minioConfig,
294+
});

0 commit comments

Comments
 (0)