From 3720cf0cac0ce50ddb4b50f695d36d72ae1ba799 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Tue, 6 Feb 2024 00:52:09 +0100 Subject: [PATCH 01/35] update deps --- package.json | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 8e7a25e..3a3bf46 100644 --- a/package.json +++ b/package.json @@ -15,32 +15,35 @@ "bugs": { "url": "https://github.com/cloudconvert/cloudconvert-node/issues" }, + "engines": { + "node": ">=16.0.0" + }, "dependencies": { - "axios": "^0.27.2", "form-data": "^4.0.0", - "socket.io-client": "^4.6.1" + "socket.io-client": "^4.7.4" }, "devDependencies": { - "@types/node": "^18.14.0", + "@types/node": "^16.18.79", "@types/socket.io-client": "^1.4.36", - "@typescript-eslint/eslint-plugin": "^5.53.0", - "@typescript-eslint/parser": "^5.53.0", - "chai": "^4.3.7", - "eslint": "^8.34.0", - "eslint-config-prettier": "^8.6.0", + "@typescript-eslint/eslint-plugin": "^6.21.0", + "@typescript-eslint/parser": "^6.21.0", + "chai": "^5.0.3", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", "eslint-config-typescript": "^3.0.0", - "eslint-plugin-prettier": "^4.2.1", + "eslint-plugin-prettier": "^5.1.3", "esm": "^3.2.25", - "mocha": "^8.4.0", - "nock": "^13.3.0", - "prettier": "^2.8.4", - "typescript": "^4.9.5" + "mocha": "^10.2.0", + "nock": "^13.5.1", + "prettier": "3.2.5", + "typescript": "^5.3.3" }, "scripts": { "prepare": "npm run build", "build": "tsc", "test": "mocha --require esm tests/unit", "test-integration": "mocha --require esm tests/integration", + "fmt": "prettier . --write", "lint": "eslint --ext .ts --ext .js --ext .json ." } } From 63c86a8d8a5b5e59520bffe8238348f8f1ddb62c Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Tue, 6 Feb 2024 00:52:46 +0100 Subject: [PATCH 02/35] update tooling and config --- .eslintignore | 2 -- .eslintrc.js | 29 ----------------------- .eslintrc.json | 17 ++++++++++++++ .github/workflows/run-tests.yml | 41 ++++++++++++++++----------------- .prettierignore | 6 +++++ .prettierrc | 14 +++++------ .vscode/settings.json | 2 +- tsconfig.json | 2 -- 8 files changed, 51 insertions(+), 62 deletions(-) delete mode 100644 .eslintrc.js create mode 100644 .eslintrc.json create mode 100644 .prettierignore diff --git a/.eslintignore b/.eslintignore index f26b6e1..c8a0d98 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,5 +1,3 @@ - .DS_Store -.idea node_modules/ built/ package-lock.json diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index dcd1e2d..0000000 --- a/.eslintrc.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = { - root: true, - env: { - node: true, - es6: true - }, - extends: [ - 'eslint:recommended', - 'plugin:@typescript-eslint/recommended', - 'prettier', - 'plugin:prettier/recommended' - ], - rules: { - 'no-console': process.env.NODE_ENV === 'production' ? 'error' : 'off', - 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', - '@typescript-eslint/no-unused-vars': 'error', - '@typescript-eslint/no-explicit-any': 0, - '@typescript-eslint/explicit-function-return-type': [ - 'error', - { - allowExpressions: true - } - ], - 'object-shorthand': ['error', 'always', { avoidQuotes: true }] - }, - parserOptions: { - parser: '@typescript-eslint/parser' - } -}; diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..e5efdc1 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,17 @@ +{ + "env": { + "browser": true, + "es2021": true, + "node": true + }, + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"], + "rules": { + "@typescript-eslint/no-explicit-any": "off" + } +} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1ff9245..f84ea92 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,28 +4,27 @@ name: Tests on: - push: - branches: [ master ] - pull_request: - branches: [ master ] + push: + branches: [master] + pull_request: + branches: [master] jobs: - build: + build: + runs-on: ubuntu-latest - runs-on: ubuntu-latest + strategy: + matrix: + node-version: [16, 18, 20] + # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ - strategy: - matrix: - node-version: [16, 18, 20] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - run: npm install -d - - run: npm run lint - - run: npm test - - run: npm run test-integration + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: npm install -d + - run: npm run lint + - run: npm test + - run: npm run test-integration diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..c8a0d98 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,6 @@ +node_modules/ +built/ +package-lock.json +package.json +tests/unit/requests/ +tests/unit/responses/ \ No newline at end of file diff --git a/.prettierrc b/.prettierrc index cbdf283..c380d6e 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,8 +1,8 @@ { - "singleQuote": true, - "trailingComma": "none", - "htmlWhitespaceSensitivity": "ignore", - "tabWidth": 4, - "semi": true, - "arrowParens": "avoid" -} \ No newline at end of file + "singleQuote": true, + "trailingComma": "none", + "htmlWhitespaceSensitivity": "ignore", + "tabWidth": 4, + "semi": true, + "arrowParens": "avoid" +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 6069cc8..a4a1864 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" } } diff --git a/tsconfig.json b/tsconfig.json index 48b582f..4430a0c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,9 @@ { "compilerOptions": { - "charset": "utf8", "declaration": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "noUncheckedIndexedAccess": true, - "keyofStringsOnly": true, "module": "commonjs", "newLine": "lf", "noFallthroughCasesInSwitch": true, From 6405d345cb3559bb9fbaee7486413a5d5e8d2663 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Tue, 6 Feb 2024 01:07:05 +0100 Subject: [PATCH 03/35] drop axios, use web fetch --- README.md | 4 +-- lib/CloudConvert.ts | 62 ++++++++++++++++++++------------- lib/JobsResource.ts | 57 +++++++++++++----------------- lib/TasksResource.ts | 82 ++++++++++++++++---------------------------- lib/UsersResource.ts | 3 +- 5 files changed, 94 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index e98d5ae..4c208a4 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ const inputFile = fs.createReadStream('./file.pdf'); await cloudConvert.tasks.upload(uploadTask, inputFile, 'file.pdf'); ``` -> **Note on custom streams**: -The length of the stream needs to be known prior to uploading. The SDK automatically detects the file size of file-based read streams. If you are using a custom stream, you need to pass a `filesize` as fourth parameter to the `upload()` method. +> **Note on custom streams**: +> The length of the stream needs to be known prior to uploading. The SDK automatically detects the file size of file-based read streams. If you are using a custom stream, you need to pass a `filesize` as fourth parameter to the `upload()` method. ## Websocket Events diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 518cf50..ac0abb2 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,14 +1,14 @@ -import axios, { type AxiosInstance } from 'axios'; import io from 'socket.io-client'; +import FormData from 'form-data'; +import { version } from '../package.json'; import JobsResource, { type JobEventData } from './JobsResource'; +import SignedUrlResource from './SignedUrlResource'; import TasksResource, { type JobTaskEventData, type TaskEventData } from './TasksResource'; import UsersResource from './UsersResource'; import WebhooksResource from './WebhooksResource'; -import { version } from '../package.json'; -import SignedUrlResource from './SignedUrlResource'; export default class CloudConvert { private socket: SocketIOClient.Socket | undefined; @@ -18,7 +18,6 @@ export default class CloudConvert { public readonly useSandbox: boolean; public readonly region: string | null; - public axios!: AxiosInstance; public tasks!: TasksResource; public jobs!: JobsResource; public users!: UsersResource; @@ -30,25 +29,6 @@ export default class CloudConvert { this.useSandbox = useSandbox; this.region = region; - this.createAxiosInstance(); - this.createResources(); - } - - createAxiosInstance(): void { - this.axios = axios.create({ - baseURL: this.useSandbox - ? 'https://api.sandbox.cloudconvert.com/v2/' - : `https://${ - this.region ? this.region + '.' : '' - }api.cloudconvert.com/v2/`, - headers: { - Authorization: `Bearer ${this.apiKey}`, - 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)` - } - }); - } - - createResources(): void { this.tasks = new TasksResource(this); this.jobs = new JobsResource(this); this.users = new UsersResource(this); @@ -56,6 +36,42 @@ export default class CloudConvert { this.signedUrls = new SignedUrlResource(); } + async call( + method: 'GET' | 'POST' | 'DELETE', + route: string, + parameters?: FormData | object + ) { + const baseURL = this.useSandbox + ? 'https://api.sandbox.cloudconvert.com/v2/' + : `https://${ + this.region ? this.region + '.' : '' + }api.cloudconvert.com/v2/`; + return await this.callWithBase(baseURL, method, route, parameters); + } + + async callWithBase( + baseURL: string, + method: 'GET' | 'POST' | 'DELETE', + route: string, + parameters?: FormData | object + ) { + const res = await fetch(new URL(route, baseURL), { + method, + headers: { + Authorization: `Bearer ${this.apiKey}`, + 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)`, + ...(parameters instanceof FormData + ? parameters.getHeaders() + : {}) + }, + body: + parameters instanceof FormData + ? parameters + : JSON.stringify(parameters) + }); + return await res.json(); + } + subscribe( channel: string, event: string, diff --git a/lib/JobsResource.ts b/lib/JobsResource.ts index ada71e4..09d8953 100644 --- a/lib/JobsResource.ts +++ b/lib/JobsResource.ts @@ -36,51 +36,40 @@ export default class JobsResource { this.cloudConvert = cloudConvert; } - async get(id: string, query = null): Promise { - const response = await this.cloudConvert.axios.get(`jobs/${id}`, { - params: query || {} - }); - return response.data.data; + async get(id: string, query?: object): Promise { + return await this.cloudConvert.call('GET', `jobs/${id}`, query); } async wait(id: string): Promise { - const response = await this.cloudConvert.axios.get(`jobs/${id}`, { - baseURL: this.cloudConvert.useSandbox - ? 'https://sync.api.sandbox.cloudconvert.com/v2/' - : `https://${ - this.cloudConvert.region - ? this.cloudConvert.region + '.' - : '' - }sync.api.cloudconvert.com/v2/` - }); - return response.data.data; + const baseURL = this.cloudConvert.useSandbox + ? 'https://sync.api.sandbox.cloudconvert.com/v2/' + : `https://${ + this.cloudConvert.region ? this.cloudConvert.region + '.' : '' + }sync.api.cloudconvert.com/v2/`; + return await this.cloudConvert.callWithBase( + baseURL, + 'GET', + `jobs/${id}` + ); } - async all( - query: { - 'filter[status]'?: JobStatus; - 'filter[tag]'?: string; - include?: string; - per_page?: number; - page?: number; - } | null = null - ): Promise { - const response = await this.cloudConvert.axios.get('jobs', { - params: query || {} - }); - return response.data.data; + async all(query?: { + 'filter[status]'?: JobStatus; + 'filter[tag]'?: string; + include?: string; + per_page?: number; + page?: number; + }): Promise { + return await this.cloudConvert.call('GET', 'jobs', query); } // See below for an explanation on how this type signature works - async create(data: JobTemplate | null = null): Promise { - const response = await this.cloudConvert.axios.post('jobs', data, { - maxBodyLength: Infinity - }); - return response.data.data; + async create(data?: JobTemplate): Promise { + return await this.cloudConvert.call('POST', 'jobs', data); } async delete(id: string): Promise { - await this.cloudConvert.axios.delete(`jobs/${id}`); + await this.cloudConvert.call('DELETE', `jobs/${id}`); } async subscribeEvent( diff --git a/lib/TasksResource.ts b/lib/TasksResource.ts index 4c197bf..39e7dda 100644 --- a/lib/TasksResource.ts +++ b/lib/TasksResource.ts @@ -1,7 +1,6 @@ import FormData, { type Stream } from 'form-data'; import CloudConvert from './CloudConvert'; import { type JobTask } from './JobsResource'; -import axios from 'axios'; import { ReadStream, statSync } from 'fs'; export type TaskEvent = 'created' | 'updated' | 'finished' | 'failed'; @@ -546,64 +545,46 @@ export default class TasksResource { this.cloudConvert = cloudConvert; } - async get( - id: string, - query: { include: string } | null = null - ): Promise { - const response = await this.cloudConvert.axios.get(`tasks/${id}`, { - params: query || {} - }); - return response.data.data; + async get(id: string, query?: { include?: string }): Promise { + return await this.cloudConvert.call('GET', `tasks/${id}`, query); } async wait(id: string): Promise { - const response = await this.cloudConvert.axios.get(`tasks/${id}`, { - baseURL: this.cloudConvert.useSandbox - ? 'https://sync.api.sandbox.cloudconvert.com/v2/' - : `https://${ - this.cloudConvert.region - ? this.cloudConvert.region + '.' - : '' - }sync.api.cloudconvert.com/v2/` - }); - return response.data.data; + const baseURL = this.cloudConvert.useSandbox + ? 'https://sync.api.sandbox.cloudconvert.com/v2/' + : `https://${ + this.cloudConvert.region ? this.cloudConvert.region + '.' : '' + }sync.api.cloudconvert.com/v2/`; + return await this.cloudConvert.callWithBase( + baseURL, + 'GET', + `tasks/${id}` + ); } async cancel(id: string): Promise { - const response = await this.cloudConvert.axios.post( - `tasks/${id}/cancel` - ); - return response.data.data; + return await this.cloudConvert.call('POST', `tasks/${id}/cancel`); } - async all( - query: { - 'filter[job_id]'?: string; - 'filter[status]'?: TaskStatus; - 'filter[operation]'?: Operation['operation']; - per_page?: number; - page?: number; - } | null = null - ): Promise { - const response = await this.cloudConvert.axios.get('tasks', { - params: query || {} - }); - return response.data.data; + async all(query?: { + 'filter[job_id]'?: string; + 'filter[status]'?: TaskStatus; + 'filter[operation]'?: Operation['operation']; + per_page?: number; + page?: number; + }): Promise { + return await this.cloudConvert.call('GET', 'tasks', query); } async create( operation: O, - data: Extract['data'] | null = null + data?: Extract['data'] ): Promise { - const response = await this.cloudConvert.axios.post( - operation, - data - ); - return response.data.data; + return await this.cloudConvert.call('POST', operation, data); } async delete(id: string): Promise { - await this.cloudConvert.axios.delete(`tasks/${id}`); + await this.cloudConvert.call('DELETE', `tasks/${id}`); } async upload( @@ -638,16 +619,11 @@ export default class TasksResource { } formData.append('file', stream, fileOptions); - return await axios.post(task.result.form.url, formData, { - maxContentLength: Infinity, - maxBodyLength: Infinity, - headers: { - ...(formData.hasKnownLength() - ? { 'Content-Length': formData.getLengthSync() } - : {}), - ...formData.getHeaders() - } - }); + return await this.cloudConvert.call( + 'POST', + task.result.form.url, + formData + ); } async subscribeEvent( diff --git a/lib/UsersResource.ts b/lib/UsersResource.ts index 07d5fbf..32ba6d9 100644 --- a/lib/UsersResource.ts +++ b/lib/UsersResource.ts @@ -18,8 +18,7 @@ export default class UsersResource { } async me(): Promise { - const response = await this.cloudConvert.axios.get('users/me'); - return response.data.data; + return await this.cloudConvert.call('GET', 'users/me'); } async subscribeJobEvent( From e7333ea8973ffdbd56fb0885cb4212df14e3b295 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Mon, 19 Feb 2024 12:21:58 +0100 Subject: [PATCH 04/35] chore: set prettier as the default formatter --- .vscode/settings.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a4a1864..a00450e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,8 @@ "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.fixAll": "explicit" - } -} + }, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + }, +} \ No newline at end of file From ea8685acf91a437b6a79fa26dc749b1d0eaa2670 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Mon, 19 Feb 2024 12:22:07 +0100 Subject: [PATCH 05/35] build: require Node 19 --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 3a3bf46..33b776f 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,14 @@ "url": "https://github.com/cloudconvert/cloudconvert-node/issues" }, "engines": { - "node": ">=16.0.0" + "node": ">=19.8.0" }, "dependencies": { "form-data": "^4.0.0", "socket.io-client": "^4.7.4" }, "devDependencies": { - "@types/node": "^16.18.79", + "@types/node": "^20.11.19", "@types/socket.io-client": "^1.4.36", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", @@ -46,4 +46,4 @@ "fmt": "prettier . --write", "lint": "eslint --ext .ts --ext .js --ext .json ." } -} +} \ No newline at end of file From b3a03c29ef8f857fedf0db0fdf4c9803dd1d0857 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Mon, 19 Feb 2024 12:50:47 +0100 Subject: [PATCH 06/35] format markdown with prettier, too --- .vscode/settings.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a00450e..96772bd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,10 @@ "editor.codeActionsOnSave": { "source.fixAll": "explicit" }, - "[typescript]": { - "editor.defaultFormatter": "esbenp.prettier-vscode", + "[md]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" }, -} \ No newline at end of file + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + } +} From a804d458116c346d2e41331b2e9b4096ceb53930 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Mon, 19 Feb 2024 12:51:13 +0100 Subject: [PATCH 07/35] move to blobs --- README.md | 5 +---- lib/CloudConvert.ts | 6 +----- lib/TasksResource.ts | 21 +++------------------ package.json | 1 - 4 files changed, 5 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 4c208a4..58f9b5e 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,11 @@ const job = await cloudConvert.jobs.create({ const uploadTask = job.tasks.filter(task => task.name === 'upload-my-file')[0]; -const inputFile = fs.createReadStream('./file.pdf'); +const inputFile = fs.openAsBlob('./file.pdf'); await cloudConvert.tasks.upload(uploadTask, inputFile, 'file.pdf'); ``` -> **Note on custom streams**: -> The length of the stream needs to be known prior to uploading. The SDK automatically detects the file size of file-based read streams. If you are using a custom stream, you need to pass a `filesize` as fourth parameter to the `upload()` method. - ## Websocket Events The node SDK can subscribe to events of the [CloudConvert socket.io API](https://cloudconvert.com/api/v2/socket#socket). diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index ac0abb2..81a8a29 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,5 +1,4 @@ import io from 'socket.io-client'; -import FormData from 'form-data'; import { version } from '../package.json'; import JobsResource, { type JobEventData } from './JobsResource'; import SignedUrlResource from './SignedUrlResource'; @@ -59,10 +58,7 @@ export default class CloudConvert { method, headers: { Authorization: `Bearer ${this.apiKey}`, - 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)`, - ...(parameters instanceof FormData - ? parameters.getHeaders() - : {}) + 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)` }, body: parameters instanceof FormData diff --git a/lib/TasksResource.ts b/lib/TasksResource.ts index 39e7dda..50abcc6 100644 --- a/lib/TasksResource.ts +++ b/lib/TasksResource.ts @@ -1,7 +1,5 @@ -import FormData, { type Stream } from 'form-data'; import CloudConvert from './CloudConvert'; import { type JobTask } from './JobsResource'; -import { ReadStream, statSync } from 'fs'; export type TaskEvent = 'created' | 'updated' | 'finished' | 'failed'; export type TaskStatus = 'waiting' | 'processing' | 'finished' | 'error'; @@ -589,9 +587,8 @@ export default class TasksResource { async upload( task: Task | JobTask, - stream: Stream, - filename: string | null = null, - size: number | null = null + stream: Blob, + filename?: string ): Promise { if (task.operation !== 'import/upload') { throw new Error('The task operation is not import/upload'); @@ -602,22 +599,10 @@ export default class TasksResource { } const formData = new FormData(); - for (const parameter in task.result.form.parameters) { formData.append(parameter, task.result.form.parameters[parameter]); } - - const fileOptions: { filename?: string; knownLength?: number } = {}; - - if (filename) { - fileOptions.filename = filename; - } - if (size) { - fileOptions.knownLength = size; - } else if (stream instanceof ReadStream) { - fileOptions.knownLength = statSync(stream.path).size; - } - formData.append('file', stream, fileOptions); + formData.append('file', stream, filename); return await this.cloudConvert.call( 'POST', diff --git a/package.json b/package.json index 33b776f..b9fabe4 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "node": ">=19.8.0" }, "dependencies": { - "form-data": "^4.0.0", "socket.io-client": "^4.7.4" }, "devDependencies": { From 7423f5d8defb6259f0c61a3444b4fb3850c688d9 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 10:18:27 +0200 Subject: [PATCH 08/35] build: require Node 20 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b9fabe4..cd7acf0 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "url": "https://github.com/cloudconvert/cloudconvert-node/issues" }, "engines": { - "node": ">=19.8.0" + "node": ">=20.11.19" }, "dependencies": { "socket.io-client": "^4.7.4" From 0d9e529bd474c928af108de63c2f528eaa17997a Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 11:25:34 +0200 Subject: [PATCH 09/35] build: update deps again --- package.json | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index cd7acf0..613829b 100644 --- a/package.json +++ b/package.json @@ -22,27 +22,29 @@ "socket.io-client": "^4.7.4" }, "devDependencies": { - "@types/node": "^20.11.19", - "@types/socket.io-client": "^1.4.36", - "@typescript-eslint/eslint-plugin": "^6.21.0", - "@typescript-eslint/parser": "^6.21.0", + "@types/chai": "^5.2.1", + "@types/mocha": "^10.0.10", + "@types/node": "^22.14.1", + "@types/socket.io-client": "^3.0.0", + "@typescript-eslint/eslint-plugin": "^8.30.1", + "@typescript-eslint/parser": "^8.30.1", "chai": "^5.0.3", - "eslint": "^8.56.0", - "eslint-config-prettier": "^9.1.0", + "eslint": "^9.24.0", + "eslint-config-prettier": "^10.1.2", "eslint-config-typescript": "^3.0.0", "eslint-plugin-prettier": "^5.1.3", - "esm": "^3.2.25", - "mocha": "^10.2.0", - "nock": "^13.5.1", - "prettier": "3.2.5", + "mocha": "^11.1.0", + "nock": "^14.0.3", + "prettier": "3.5.3", + "tsx": "^4.19.3", "typescript": "^5.3.3" }, "scripts": { "prepare": "npm run build", "build": "tsc", - "test": "mocha --require esm tests/unit", - "test-integration": "mocha --require esm tests/integration", + "test": "mocha --require tsx tests/unit/*.ts", + "test-integration": "mocha --require tsx tests/integration/*.ts", "fmt": "prettier . --write", "lint": "eslint --ext .ts --ext .js --ext .json ." } -} \ No newline at end of file +} From a8c1a654bb570b26ced8986caef26f21bd3f6672 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 11:25:44 +0200 Subject: [PATCH 10/35] style: reformat README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 58f9b5e..3090e64 100644 --- a/README.md +++ b/README.md @@ -239,5 +239,5 @@ and even auto-fix as many things as possible by running ## Resources -- [API v2 Documentation](https://cloudconvert.com/api/v2) -- [CloudConvert Blog](https://cloudconvert.com/blog) +- [API v2 Documentation](https://cloudconvert.com/api/v2) +- [CloudConvert Blog](https://cloudconvert.com/blog) From 0d7b31ac59b8e771a5467586b83da6be392a00aa Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 11:25:56 +0200 Subject: [PATCH 11/35] style: reformat tsconfig --- tsconfig.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tsconfig.json b/tsconfig.json index 4430a0c..0f67ce5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,11 @@ "strict": true, "resolveJsonModule": true, "target": "es6", - "typeRoots": ["./node_modules/@types"] + "typeRoots": [ + "./node_modules/@types" + ] }, - "include": ["./lib/*"] -} + "include": [ + "./lib/*", + ] +} \ No newline at end of file From 39144ac6497be07b6e319efefa5e15742389e0de Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 11:26:34 +0200 Subject: [PATCH 12/35] fix: update usage of fetch and socket.io --- lib/CloudConvert.ts | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 81a8a29..5153594 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,4 +1,4 @@ -import io from 'socket.io-client'; +import { io, type Socket } from 'socket.io-client'; import { version } from '../package.json'; import JobsResource, { type JobEventData } from './JobsResource'; import SignedUrlResource from './SignedUrlResource'; @@ -10,7 +10,7 @@ import UsersResource from './UsersResource'; import WebhooksResource from './WebhooksResource'; export default class CloudConvert { - private socket: SocketIOClient.Socket | undefined; + private socket: Socket | undefined; private subscribedChannels: Map | undefined; public readonly apiKey: string; @@ -54,18 +54,36 @@ export default class CloudConvert { route: string, parameters?: FormData | object ) { - const res = await fetch(new URL(route, baseURL), { + const url = new URL(route, baseURL); + let body: RequestInit['body'] | undefined; + if (parameters instanceof FormData) { + body = parameters; + } else { + if (method === 'GET') { + url.search = new URLSearchParams( + Object.entries(parameters ?? {}) + ).toString(); + } else { + body = JSON.stringify(parameters); + } + } + const res = await fetch(url, { method, headers: { Authorization: `Bearer ${this.apiKey}`, 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)` }, - body: - parameters instanceof FormData - ? parameters - : JSON.stringify(parameters) + body }); - return await res.json(); + if ( + !res.ok || + res.headers.get('Content-Type')?.toLowerCase() !== + 'application/json' + ) { + return undefined; + } + const { data } = await res.json(); + return data; } subscribe( @@ -77,13 +95,11 @@ export default class CloudConvert { | ((event: JobTaskEventData) => void) ): void { if (!this.socket) { - this.socket = io.connect( + this.socket = io( this.useSandbox ? 'https://socketio.sandbox.cloudconvert.com' : 'https://socketio.cloudconvert.com', - { - transports: ['websocket'] - } + { transports: ['websocket'] } ); this.subscribedChannels = new Map(); } @@ -91,11 +107,7 @@ export default class CloudConvert { if (!this.subscribedChannels?.get(channel)) { this.socket.emit('subscribe', { channel, - auth: { - headers: { - Authorization: `Bearer ${this.apiKey}` - } - } + auth: { headers: { Authorization: `Bearer ${this.apiKey}` } } }); this.subscribedChannels?.set(channel, true); } From e58a9aa2b858b249d15d48a4af9c847c68ca541b Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 11:27:09 +0200 Subject: [PATCH 13/35] test: migrate tests to TS --- tests/integration/{ApiKey.js => ApiKey.ts} | 0 ...obsResourceTest.js => JobsResourceTest.ts} | 24 ++++----------- ...ksResourceTest.js => TasksResourceTest.ts} | 2 +- ...rsResourceTest.js => UsersResourceTest.ts} | 2 +- ...obsResourceTest.js => JobsResourceTest.ts} | 29 ++++++++----------- ...sourceTest.js => SignedUrlResourceTest.ts} | 8 +++-- ...ksResourceTest.js => TasksResourceTest.ts} | 28 +++++++++--------- ...rsResourceTest.js => UsersResourceTest.ts} | 6 ++-- ...esourceTest.js => WebhooksResourceTest.ts} | 19 +++++------- 9 files changed, 49 insertions(+), 69 deletions(-) rename tests/integration/{ApiKey.js => ApiKey.ts} (100%) rename tests/integration/{JobsResourceTest.js => JobsResourceTest.ts} (81%) rename tests/integration/{TasksResourceTest.js => TasksResourceTest.ts} (96%) rename tests/integration/{UsersResourceTest.js => UsersResourceTest.ts} (94%) rename tests/unit/{JobsResourceTest.js => JobsResourceTest.ts} (79%) rename tests/unit/{SignedUrlResourceTest.js => SignedUrlResourceTest.ts} (88%) rename tests/unit/{TasksResourceTest.js => TasksResourceTest.ts} (81%) rename tests/unit/{UsersResourceTest.js => UsersResourceTest.ts} (82%) rename tests/unit/{WebhooksResourceTest.js => WebhooksResourceTest.ts} (63%) diff --git a/tests/integration/ApiKey.js b/tests/integration/ApiKey.ts similarity index 100% rename from tests/integration/ApiKey.js rename to tests/integration/ApiKey.ts diff --git a/tests/integration/JobsResourceTest.js b/tests/integration/JobsResourceTest.ts similarity index 81% rename from tests/integration/JobsResourceTest.js rename to tests/integration/JobsResourceTest.ts index 7dcd87f..1eba19b 100644 --- a/tests/integration/JobsResourceTest.js +++ b/tests/integration/JobsResourceTest.ts @@ -2,7 +2,7 @@ import CloudConvert from '../../built/lib/CloudConvert.js'; import { assert } from 'chai'; import * as fs from 'fs'; import * as os from 'os'; -import apiKey from './ApiKey'; +import apiKey from './ApiKey.js'; import axios from 'axios'; describe('JobsResource', () => { @@ -19,13 +19,8 @@ describe('JobsResource', () => { let job = await this.cloudConvert.jobs.create({ tag: 'integration-test-upload-download', tasks: { - 'import-it': { - operation: 'import/upload' - }, - 'export-it': { - input: 'import-it', - operation: 'export/url' - } + 'import-it': { operation: 'import/upload' }, + 'export-it': { input: 'import-it', operation: 'export/url' } } }); @@ -50,9 +45,7 @@ describe('JobsResource', () => { const writer = fs.createWriteStream(this.tmpPath); - const response = await axios(file.url, { - responseType: 'stream' - }); + const response = await axios(file.url, { responseType: 'stream' }); response.data.pipe(writer); @@ -79,13 +72,8 @@ describe('JobsResource', () => { let job = await this.cloudConvert.jobs.create({ tag: 'integration-test-socket', tasks: { - 'import-it': { - operation: 'import/upload' - }, - 'export-it': { - input: 'import-it', - operation: 'export/url' - } + 'import-it': { operation: 'import/upload' }, + 'export-it': { input: 'import-it', operation: 'export/url' } } }); diff --git a/tests/integration/TasksResourceTest.js b/tests/integration/TasksResourceTest.ts similarity index 96% rename from tests/integration/TasksResourceTest.js rename to tests/integration/TasksResourceTest.ts index e30fd5b..4e3ad38 100644 --- a/tests/integration/TasksResourceTest.js +++ b/tests/integration/TasksResourceTest.ts @@ -1,7 +1,7 @@ import CloudConvert from '../../built/lib/CloudConvert.js'; import { assert } from 'chai'; import * as fs from 'fs'; -import apiKey from './ApiKey'; +import apiKey from './ApiKey.js'; describe('TasksResource', () => { beforeEach(() => { diff --git a/tests/integration/UsersResourceTest.js b/tests/integration/UsersResourceTest.ts similarity index 94% rename from tests/integration/UsersResourceTest.js rename to tests/integration/UsersResourceTest.ts index 9230c80..9bf52dd 100644 --- a/tests/integration/UsersResourceTest.js +++ b/tests/integration/UsersResourceTest.ts @@ -1,5 +1,5 @@ import CloudConvert from '../../built/lib/CloudConvert.js'; -import apiKey from './ApiKey'; +import apiKey from './ApiKey.js'; import { assert } from 'chai'; describe('UsersResource', () => { diff --git a/tests/unit/JobsResourceTest.js b/tests/unit/JobsResourceTest.ts similarity index 79% rename from tests/unit/JobsResourceTest.js rename to tests/unit/JobsResourceTest.ts index e1b37bd..99f63e0 100644 --- a/tests/unit/JobsResourceTest.js +++ b/tests/unit/JobsResourceTest.ts @@ -3,23 +3,23 @@ import { assert } from 'chai'; import nock from 'nock'; describe('JobsResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert('test'); + cloudConvert = new CloudConvert('test'); }); describe('all()', () => { it('should fetch all jobs', async () => { nock('https://api.cloudconvert.com', { - reqheaders: { - Authorization: 'Bearer test' - } + reqheaders: { Authorization: 'Bearer test' } }) .get('/v2/jobs') .replyWithFile(200, __dirname + '/responses/jobs.json', { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.jobs.all(); + const data = await cloudConvert.jobs.all(); assert.isArray(data); assert.equal(data[0].id, 'bd7d06b4-60fb-472b-b3a3-9034b273df07'); @@ -36,7 +36,7 @@ describe('JobsResource', () => { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.jobs.get( + const data = await cloudConvert.jobs.get( 'cd82535b-0614-4b23-bbba-b24ab0e892f7' ); @@ -48,15 +48,12 @@ describe('JobsResource', () => { describe('create()', () => { it('should send the create request', async () => { nock('https://api.cloudconvert.com') - .post('/v2/jobs', { - tag: 'test', - tasks: {} - }) + .post('/v2/jobs', { tag: 'test', tasks: {} }) .replyWithFile(200, __dirname + '/responses/job_created.json', { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.jobs.create({ + const data = await cloudConvert.jobs.create({ tag: 'test', tasks: {} }); @@ -72,7 +69,7 @@ describe('JobsResource', () => { .delete('/v2/jobs/2f901289-c9fe-4c89-9c4b-98be526bdfbf') .reply(204); - await this.cloudConvert.jobs.delete( + await cloudConvert.jobs.delete( '2f901289-c9fe-4c89-9c4b-98be526bdfbf' ); }); @@ -86,17 +83,15 @@ describe('JobsResource', () => { .replyWithFile( 200, __dirname + '/responses/job_finished.json', - { - 'Content-Type': 'application/json' - } + { 'Content-Type': 'application/json' } ); - const job = await this.cloudConvert.jobs.get( + const job = await cloudConvert.jobs.get( 'b2e4eb2b-a744-4da2-97cd-776d393532a8', { include: 'tasks' } ); - const exportUrls = this.cloudConvert.jobs.getExportUrls(job); + const exportUrls = cloudConvert.jobs.getExportUrls(job); assert.isArray(exportUrls); assert.lengthOf(exportUrls, 1); diff --git a/tests/unit/SignedUrlResourceTest.js b/tests/unit/SignedUrlResourceTest.ts similarity index 88% rename from tests/unit/SignedUrlResourceTest.js rename to tests/unit/SignedUrlResourceTest.ts index 4b91f39..12f336d 100644 --- a/tests/unit/SignedUrlResourceTest.js +++ b/tests/unit/SignedUrlResourceTest.ts @@ -2,8 +2,10 @@ import CloudConvert from '../../built/lib/CloudConvert.js'; import { assert } from 'chai'; describe('SignedUrlResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert('test'); + cloudConvert = new CloudConvert('test'); }); describe('create()', () => { @@ -25,9 +27,9 @@ describe('SignedUrlResource', () => { inline: true } } - }; + } as const; - const url = this.cloudConvert.signedUrls.sign( + const url = cloudConvert.signedUrls.sign( base, signingSecret, job, diff --git a/tests/unit/TasksResourceTest.js b/tests/unit/TasksResourceTest.ts similarity index 81% rename from tests/unit/TasksResourceTest.js rename to tests/unit/TasksResourceTest.ts index 0bbcf5b..1f334ff 100644 --- a/tests/unit/TasksResourceTest.js +++ b/tests/unit/TasksResourceTest.ts @@ -4,23 +4,23 @@ import * as fs from 'fs'; import nock from 'nock'; describe('TasksResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert('test'); + cloudConvert = new CloudConvert('test'); }); describe('all()', () => { it('should fetch all tasks', async () => { nock('https://api.cloudconvert.com', { - reqheaders: { - Authorization: 'Bearer test' - } + reqheaders: { Authorization: 'Bearer test' } }) .get('/v2/tasks') .replyWithFile(200, __dirname + '/responses/tasks.json', { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.tasks.all(); + const data = await cloudConvert.tasks.all(); assert.isArray(data); assert.equal(data[0].id, '73df1e16-fd8b-47a1-a156-f197babde91a'); @@ -36,7 +36,7 @@ describe('TasksResource', () => { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.tasks.get( + const data = await cloudConvert.tasks.get( '4c80f1ae-5b3a-43d5-bb58-1a5c4eb4e46b' ); @@ -59,7 +59,7 @@ describe('TasksResource', () => { { 'Content-Type': 'application/json' } ); - const data = await this.cloudConvert.tasks.create('convert', { + const data = await cloudConvert.tasks.create('convert', { name: 'test', url: 'http://invalid.url', filename: 'test.file' @@ -76,7 +76,7 @@ describe('TasksResource', () => { .delete('/v2/tasks/2f901289-c9fe-4c89-9c4b-98be526bdfbf') .reply(204); - await this.cloudConvert.tasks.delete( + await cloudConvert.tasks.delete( '2f901289-c9fe-4c89-9c4b-98be526bdfbf' ); }); @@ -85,30 +85,28 @@ describe('TasksResource', () => { describe('upload()', () => { it('should send the upload request', async () => { nock('https://api.cloudconvert.com') - .post('/v2/import/upload', {}) + .post('/v2/import/upload') .replyWithFile( 200, __dirname + '/responses/upload_task_created.json', { 'Content-Type': 'application/json' } ); - const task = await this.cloudConvert.tasks.create('import/upload'); + const task = await cloudConvert.tasks.create('import/upload'); nock('https://upload.sandbox.cloudconvert.com', { - reqheaders: { - 'Content-Type': /multipart\/form-data/i - } + reqheaders: { 'Content-Type': /multipart\/form-data/i } }) .post( '/storage.de1.cloud.ovh.net/v1/AUTH_b2cffe8f45324c2bba39e8db1aedb58f/cloudconvert-files-sandbox/8aefdb39-34c8-4c7a-9f2e-1751686d615e/?s=jNf7hn3zox1iZfZY6NirNA&e=1559588529' ) .reply(201); - const stream = fs.createReadStream( + const blob = await fs.openAsBlob( __dirname + '/../integration/files/input.png' ); - await this.cloudConvert.tasks.upload(task, stream); + await cloudConvert.tasks.upload(task, blob); }); }); }); diff --git a/tests/unit/UsersResourceTest.js b/tests/unit/UsersResourceTest.ts similarity index 82% rename from tests/unit/UsersResourceTest.js rename to tests/unit/UsersResourceTest.ts index 5aa192d..ca1785b 100644 --- a/tests/unit/UsersResourceTest.js +++ b/tests/unit/UsersResourceTest.ts @@ -3,8 +3,10 @@ import { assert } from 'chai'; import nock from 'nock'; describe('UsersResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert('test'); + cloudConvert = new CloudConvert('test'); }); describe('me()', () => { @@ -15,7 +17,7 @@ describe('UsersResource', () => { 'Content-Type': 'application/json' }); - const data = await this.cloudConvert.users.me(); + const data = await cloudConvert.users.me(); assert.isObject(data); assert.equal(data.id, 1); diff --git a/tests/unit/WebhooksResourceTest.js b/tests/unit/WebhooksResourceTest.ts similarity index 63% rename from tests/unit/WebhooksResourceTest.js rename to tests/unit/WebhooksResourceTest.ts index 982a1dd..897d5be 100644 --- a/tests/unit/WebhooksResourceTest.js +++ b/tests/unit/WebhooksResourceTest.ts @@ -3,8 +3,10 @@ import { assert } from 'chai'; import * as fs from 'fs'; describe('WebhooksResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert('test'); + cloudConvert = new CloudConvert('test'); }); describe('verify()', () => { @@ -13,22 +15,15 @@ describe('WebhooksResource', () => { const signature = '576b653f726c85265a389532988f483b5c7d7d5f40cede5f5ddf9c3f02934f35'; const payloadString = fs.readFileSync( - __dirname + '/requests/webhook_job_finished_payload.json' + __dirname + '/requests/webhook_job_finished_payload.json', + 'utf-8' ); assert.isFalse( - this.cloudConvert.webhooks.verify( - payloadString, - 'invalid', - secret - ) + cloudConvert.webhooks.verify(payloadString, 'invalid', secret) ); assert.isTrue( - this.cloudConvert.webhooks.verify( - payloadString, - signature, - secret - ) + cloudConvert.webhooks.verify(payloadString, signature, secret) ); }); }); From 621db1db273cecd745580521acd2b863b1010b22 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:04:58 +0200 Subject: [PATCH 14/35] build: use minimal supported Node types --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 613829b..25d209d 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "devDependencies": { "@types/chai": "^5.2.1", "@types/mocha": "^10.0.10", - "@types/node": "^22.14.1", + "@types/node": "^20.11.19", "@types/socket.io-client": "^3.0.0", "@typescript-eslint/eslint-plugin": "^8.30.1", "@typescript-eslint/parser": "^8.30.1", @@ -47,4 +47,4 @@ "fmt": "prettier . --write", "lint": "eslint --ext .ts --ext .js --ext .json ." } -} +} \ No newline at end of file From 2a75f8b5936d0230e7d64f441f826e2fa7d02e10 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:05:21 +0200 Subject: [PATCH 15/35] feat: support many types of files --- lib/CloudConvert.ts | 150 +++++++++++++++++++++++++++++++++++++------ lib/TasksResource.ts | 14 ++-- 2 files changed, 140 insertions(+), 24 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 5153594..a207e72 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,4 +1,5 @@ import { io, type Socket } from 'socket.io-client'; +import { Readable } from 'node:stream'; import { version } from '../package.json'; import JobsResource, { type JobEventData } from './JobsResource'; import SignedUrlResource from './SignedUrlResource'; @@ -9,6 +10,85 @@ import TasksResource, { import UsersResource from './UsersResource'; import WebhooksResource from './WebhooksResource'; +export type UploadFileSource = + | Blob + | Uint8Array + | Iterable + | AsyncIterable + | NodeJS.ReadableStream; + +export class UploadFile { + private readonly attributes: Array<[key: string, value: unknown]> = []; + private readonly data: AsyncIterable; + constructor( + data: UploadFileSource, + private readonly filename?: string + ) { + this.data = UploadFile.unifySources(data); + } + add(key: string, value: unknown) { + this.attributes.push([key, value]); + } + async *stream() { + const enc = new TextEncoder(); + const boundary = `----------${Array.from(Array(32)) + .map(() => Math.random().toString(36)[2] || 0) + .join('')}`; + // Start multipart/form-data protocol + yield enc.encode(`--${boundary}\r\n`); + // Send all attributes + const separator = enc.encode(`\r\n--${boundary}\r\n`); + let first = true; + for (const [key, value] of this.attributes) { + if (value == null) continue; + if (!first) yield separator; + yield enc.encode( + `content-disposition:form-data;name="${key}"\r\n\r\n${value}` + ); + first = false; + } + // Send file + if (!first) yield separator; + yield enc.encode( + `content-disposition:form-data;name="file";filename=${this.filename}\r\ncontent-type:application/octet-stream\r\n\r\n` + ); + yield* this.data; + // End multipart/form-data protocol + yield enc.encode(`\r\n--${boundary}--\r\n`); + } + + static async *unifySources( + data: UploadFileSource + ): AsyncIterable { + if (data instanceof Uint8Array) { + yield data; + return; + } + + if (data instanceof Blob) { + yield data.bytes(); + return; + } + + if (Symbol.iterator in data) { + yield* data; + return; + } + + if (Symbol.asyncIterator in data) { + const it = data[Symbol.asyncIterator](); + for await (const chunk of data) { + if (typeof chunk === 'string') + throw new Error( + 'bad file data, received string but expected Uint8Array' + ); + yield chunk; + } + return; + } + } +} + export default class CloudConvert { private socket: Socket | undefined; private subscribedChannels: Map | undefined; @@ -38,7 +118,7 @@ export default class CloudConvert { async call( method: 'GET' | 'POST' | 'DELETE', route: string, - parameters?: FormData | object + parameters?: UploadFile | object ) { const baseURL = this.useSandbox ? 'https://api.sandbox.cloudconvert.com/v2/' @@ -52,28 +132,27 @@ export default class CloudConvert { baseURL: string, method: 'GET' | 'POST' | 'DELETE', route: string, - parameters?: FormData | object + parameters?: UploadFile | object ) { const url = new URL(route, baseURL); - let body: RequestInit['body'] | undefined; - if (parameters instanceof FormData) { - body = parameters; - } else { - if (method === 'GET') { - url.search = new URLSearchParams( - Object.entries(parameters ?? {}) - ).toString(); - } else { - body = JSON.stringify(parameters); - } + const { contentType, search, body } = prepareParameters( + method, + parameters + ); + if (search !== undefined) { + url.search = search; } + const headers = { + Authorization: `Bearer ${this.apiKey}`, + 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)`, + ...(contentType ? { 'Content-Type': contentType } : {}) + }; const res = await fetch(url, { method, - headers: { - Authorization: `Bearer ${this.apiKey}`, - 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)` - }, - body + headers, + body, + // @ts-expect-error incorrect types in @types/node@20 + duplex: 'half' }); if ( !res.ok || @@ -127,3 +206,38 @@ export default class CloudConvert { this.socket?.close(); } } + +function prepareParameters( + method: 'GET' | 'POST' | 'DELETE', + data?: UploadFile | object +): { + contentType?: string; + body?: string | ReadableStream; + search?: string; +} { + if (data === undefined) { + return {}; + } + + if (method === 'GET') { + // abort early if all data needs to go into the search params + const entries = Object.entries(data ?? {}); + return { search: new URLSearchParams(entries).toString() }; + } + + if (data instanceof UploadFile) { + return { + contentType: 'multipart/form-data', + body: asyncIterableToReadableStream(data.stream()) + }; + } + + return { contentType: 'application/json', body: JSON.stringify(data) }; +} + +function asyncIterableToReadableStream( + it: AsyncIterable +): ReadableStream { + const r = Readable.from(it); + return Readable.toWeb(r) as ReadableStream; +} diff --git a/lib/TasksResource.ts b/lib/TasksResource.ts index 50abcc6..c94493c 100644 --- a/lib/TasksResource.ts +++ b/lib/TasksResource.ts @@ -1,4 +1,7 @@ -import CloudConvert from './CloudConvert'; +import CloudConvert, { + UploadFile, + type UploadFileSource +} from './CloudConvert'; import { type JobTask } from './JobsResource'; export type TaskEvent = 'created' | 'updated' | 'finished' | 'failed'; @@ -587,7 +590,7 @@ export default class TasksResource { async upload( task: Task | JobTask, - stream: Blob, + stream: UploadFileSource, filename?: string ): Promise { if (task.operation !== 'import/upload') { @@ -598,16 +601,15 @@ export default class TasksResource { throw new Error('The task is not ready for uploading'); } - const formData = new FormData(); + const uploadFile = new UploadFile(stream, filename); for (const parameter in task.result.form.parameters) { - formData.append(parameter, task.result.form.parameters[parameter]); + uploadFile.add(parameter, task.result.form.parameters[parameter]); } - formData.append('file', stream, filename); return await this.cloudConvert.call( 'POST', task.result.form.url, - formData + uploadFile ); } From 1748a08fed6452cefd259144a487ea0627268f49 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:07:03 +0200 Subject: [PATCH 16/35] docs: revert temporary changes to file input source support --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3090e64..e2294df 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ const job = await cloudConvert.jobs.create({ const uploadTask = job.tasks.filter(task => task.name === 'upload-my-file')[0]; -const inputFile = fs.openAsBlob('./file.pdf'); +const inputFile = fs.createReadStream('./file.pdf'); await cloudConvert.tasks.upload(uploadTask, inputFile, 'file.pdf'); ``` From e9c860eb5ca5c72ac052b7fdf9947f02fc93e41c Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:42:56 +0200 Subject: [PATCH 17/35] chore: migrate ESLint config --- .eslintignore | 6 --- .eslintrc.json | 17 -------- eslint.config.mjs | 54 ++++++++++++++++++++++++++ lib/CloudConvert.ts | 1 - package.json | 3 ++ tests/integration/JobsResourceTest.ts | 37 +++++++++--------- tests/integration/TasksResourceTest.ts | 12 +++--- tests/integration/UsersResourceTest.ts | 6 ++- tsconfig.json | 10 ++--- 9 files changed, 89 insertions(+), 57 deletions(-) delete mode 100644 .eslintignore delete mode 100644 .eslintrc.json create mode 100644 eslint.config.mjs diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index c8a0d98..0000000 --- a/.eslintignore +++ /dev/null @@ -1,6 +0,0 @@ -node_modules/ -built/ -package-lock.json -package.json -tests/unit/requests/ -tests/unit/responses/ \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index e5efdc1..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "env": { - "browser": true, - "es2021": true, - "node": true - }, - "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "plugins": ["@typescript-eslint"], - "rules": { - "@typescript-eslint/no-explicit-any": "off" - } -} diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..7949ffc --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,54 @@ +import { defineConfig, globalIgnores } from 'eslint/config'; +import typescriptEslint from '@typescript-eslint/eslint-plugin'; +import globals from 'globals'; +import tsParser from '@typescript-eslint/parser'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all +}); + +export default defineConfig([ + globalIgnores([ + '**/node_modules/', + '**/built/', + '**/package-lock.json', + '**/package.json', + 'tests/unit/requests/', + 'tests/unit/responses/', + '.vscode/', + 'tsconfig.json' + ]), + { + extends: compat.extends( + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended' + ), + + plugins: { + '@typescript-eslint': typescriptEslint + }, + + languageOptions: { + globals: { + ...globals.browser, + ...globals.node + }, + + parser: tsParser, + ecmaVersion: 'latest', + sourceType: 'module' + }, + + rules: { + '@typescript-eslint/no-explicit-any': 'off' + } + } +]); diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index a207e72..87bb8b9 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -76,7 +76,6 @@ export class UploadFile { } if (Symbol.asyncIterator in data) { - const it = data[Symbol.asyncIterator](); for await (const chunk of data) { if (typeof chunk === 'string') throw new Error( diff --git a/package.json b/package.json index 76c1f3c..12538e3 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,8 @@ "socket.io-client": "^4.7.4" }, "devDependencies": { + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "^9.24.0", "@types/chai": "^5.2.1", "@types/mocha": "^10.0.10", "@types/node": "^20.11.19", @@ -33,6 +35,7 @@ "eslint-config-prettier": "^10.1.2", "eslint-config-typescript": "^3.0.0", "eslint-plugin-prettier": "^5.1.3", + "globals": "^16.0.0", "mocha": "^11.1.0", "nock": "^14.0.3", "prettier": "3.5.3", diff --git a/tests/integration/JobsResourceTest.ts b/tests/integration/JobsResourceTest.ts index 1eba19b..c2fb0b5 100644 --- a/tests/integration/JobsResourceTest.ts +++ b/tests/integration/JobsResourceTest.ts @@ -6,17 +6,20 @@ import apiKey from './ApiKey.js'; import axios from 'axios'; describe('JobsResource', () => { + let cloudConvert: CloudConvert; + let tmpPath: string; + beforeEach(() => { - this.cloudConvert = new CloudConvert(apiKey, true); + cloudConvert = new CloudConvert(apiKey, true); }); describe('create()', () => { beforeEach(() => { - this.tmpPath = os.tmpdir() + '/tmp.png'; + tmpPath = os.tmpdir() + '/tmp.png'; }); it('test upload and download files', async () => { - let job = await this.cloudConvert.jobs.create({ + let job = await cloudConvert.jobs.create({ tag: 'integration-test-upload-download', tasks: { 'import-it': { operation: 'import/upload' }, @@ -32,18 +35,18 @@ describe('JobsResource', () => { __dirname + '/../integration/files/input.png' ); - await this.cloudConvert.tasks.upload(uploadTask, stream); + await cloudConvert.tasks.upload(uploadTask, stream); - job = await this.cloudConvert.jobs.wait(job.id); + job = await cloudConvert.jobs.wait(job.id); assert.equal(job.status, 'finished'); // download export file - const file = this.cloudConvert.jobs.getExportUrls(job)[0]; + const file = cloudConvert.jobs.getExportUrls(job)[0]; assert.equal(file.filename, 'input.png'); - const writer = fs.createWriteStream(this.tmpPath); + const writer = fs.createWriteStream(tmpPath); const response = await axios(file.url, { responseType: 'stream' }); @@ -55,21 +58,21 @@ describe('JobsResource', () => { }); // check file size - const stat = fs.statSync(this.tmpPath); + const stat = fs.statSync(tmpPath); assert.equal(stat.size, 46937); - await this.cloudConvert.jobs.delete(job.id); + await cloudConvert.jobs.delete(job.id); }).timeout(30000); afterEach(() => { - fs.unlinkSync(this.tmpPath); + fs.unlinkSync(tmpPath); }); }); describe('subscribeEvent()', () => { it('test listening for finished event', async () => { - let job = await this.cloudConvert.jobs.create({ + const job = await cloudConvert.jobs.create({ tag: 'integration-test-socket', tasks: { 'import-it': { operation: 'import/upload' }, @@ -85,23 +88,19 @@ describe('JobsResource', () => { __dirname + '/../integration/files/input.png' ); - this.cloudConvert.tasks.upload(uploadTask, stream); + cloudConvert.tasks.upload(uploadTask, stream); const event = await new Promise(resolve => { - this.cloudConvert.jobs.subscribeEvent( - job.id, - 'finished', - resolve - ); + cloudConvert.jobs.subscribeEvent(job.id, 'finished', resolve); }); assert.equal(event.job.status, 'finished'); - await this.cloudConvert.jobs.delete(job.id); + await cloudConvert.jobs.delete(job.id); }).timeout(30000); afterEach(() => { - this.cloudConvert.closeSocket(); + cloudConvert.closeSocket(); }); }); }); diff --git a/tests/integration/TasksResourceTest.ts b/tests/integration/TasksResourceTest.ts index 4e3ad38..9570be8 100644 --- a/tests/integration/TasksResourceTest.ts +++ b/tests/integration/TasksResourceTest.ts @@ -4,13 +4,15 @@ import * as fs from 'fs'; import apiKey from './ApiKey.js'; describe('TasksResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert(apiKey, true); + cloudConvert = new CloudConvert(apiKey, true); }); describe('upload()', () => { it('uploads input.png', async () => { - let task = await this.cloudConvert.tasks.create('import/upload', { + let task = await cloudConvert.tasks.create('import/upload', { name: 'upload-test' }); @@ -18,14 +20,14 @@ describe('TasksResource', () => { __dirname + '/../integration/files/input.png' ); - await this.cloudConvert.tasks.upload(task, stream); + await cloudConvert.tasks.upload(task, stream); - task = await this.cloudConvert.tasks.wait(task.id); + task = await cloudConvert.tasks.wait(task.id); assert.equal(task.status, 'finished'); assert.equal(task.result.files[0].filename, 'input.png'); - await this.cloudConvert.tasks.delete(task.id); + await cloudConvert.tasks.delete(task.id); }).timeout(30000); }); }); diff --git a/tests/integration/UsersResourceTest.ts b/tests/integration/UsersResourceTest.ts index 9bf52dd..a1d3df0 100644 --- a/tests/integration/UsersResourceTest.ts +++ b/tests/integration/UsersResourceTest.ts @@ -3,13 +3,15 @@ import apiKey from './ApiKey.js'; import { assert } from 'chai'; describe('UsersResource', () => { + let cloudConvert: CloudConvert; + beforeEach(() => { - this.cloudConvert = new CloudConvert(apiKey, true); + cloudConvert = new CloudConvert(apiKey, true); }); describe('me()', () => { it('should fetch the current user', async () => { - const data = await this.cloudConvert.users.me(); + const data = await cloudConvert.users.me(); console.log(data); diff --git a/tsconfig.json b/tsconfig.json index 0f67ce5..4430a0c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,11 +17,7 @@ "strict": true, "resolveJsonModule": true, "target": "es6", - "typeRoots": [ - "./node_modules/@types" - ] + "typeRoots": ["./node_modules/@types"] }, - "include": [ - "./lib/*", - ] -} \ No newline at end of file + "include": ["./lib/*"] +} From 9db83448c87e1469999ea98c2985c8e17df2c20a Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:44:03 +0200 Subject: [PATCH 18/35] chore: change Node versions in CI --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f84ea92..1a54b0b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - node-version: [16, 18, 20] + node-version: [20, 22, 24] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 463943d29bef3422b2c59ffe2c125c5c0990f506 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:45:05 +0200 Subject: [PATCH 19/35] chore: do not use Node 24 yet --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1a54b0b..03c4ab2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - node-version: [20, 22, 24] + node-version: [20, 22] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 766a8145df1c768fabeb2a0e5e792a1b08fbeb56 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 16:53:11 +0200 Subject: [PATCH 20/35] test: migrate e2e tests away from axios --- tests/integration/JobsResourceTest.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/JobsResourceTest.ts b/tests/integration/JobsResourceTest.ts index c2fb0b5..2a9eaaf 100644 --- a/tests/integration/JobsResourceTest.ts +++ b/tests/integration/JobsResourceTest.ts @@ -1,9 +1,10 @@ import CloudConvert from '../../built/lib/CloudConvert.js'; import { assert } from 'chai'; import * as fs from 'fs'; +import { Readable } from 'node:stream'; +import { type ReadableStream } from 'node:stream/web'; import * as os from 'os'; import apiKey from './ApiKey.js'; -import axios from 'axios'; describe('JobsResource', () => { let cloudConvert: CloudConvert; @@ -48,11 +49,11 @@ describe('JobsResource', () => { const writer = fs.createWriteStream(tmpPath); - const response = await axios(file.url, { responseType: 'stream' }); + const response = (await fetch(file.url!)).body as ReadableStream; - response.data.pipe(writer); + Readable.fromWeb(response).pipe(writer); - await new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { writer.on('finish', resolve); writer.on('error', reject); }); From 7d62e979f7d6f595a07287a6f43f1b69d36c1270 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 16 Apr 2025 17:10:17 +0200 Subject: [PATCH 21/35] fix: throw better errors --- lib/CloudConvert.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 87bb8b9..45b1e6c 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -153,10 +153,14 @@ export default class CloudConvert { // @ts-expect-error incorrect types in @types/node@20 duplex: 'half' }); + if (!res.ok) { + // @ts-expect-error cause not present in types yet + throw new Error(res.statusText, { cause: res }); + } + if ( - !res.ok || res.headers.get('Content-Type')?.toLowerCase() !== - 'application/json' + 'application/json' ) { return undefined; } From e78c0c4b1e6c223f20260cb12909afcb6d2bbb52 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sat, 19 Apr 2025 18:03:53 +0200 Subject: [PATCH 22/35] feat: add temporary logging --- lib/CloudConvert.ts | 2 ++ tests/integration/TasksResourceTest.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 45b1e6c..5575ff7 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -154,6 +154,8 @@ export default class CloudConvert { duplex: 'half' }); if (!res.ok) { + console.error('SND:', url, method, headers); + console.error('RCV:', res, await res.text()); // @ts-expect-error cause not present in types yet throw new Error(res.statusText, { cause: res }); } diff --git a/tests/integration/TasksResourceTest.ts b/tests/integration/TasksResourceTest.ts index 9570be8..9ee7fca 100644 --- a/tests/integration/TasksResourceTest.ts +++ b/tests/integration/TasksResourceTest.ts @@ -16,18 +16,22 @@ describe('TasksResource', () => { name: 'upload-test' }); + console.log('task', task); const stream = fs.createReadStream( __dirname + '/../integration/files/input.png' ); - await cloudConvert.tasks.upload(task, stream); + const res = await cloudConvert.tasks.upload(task, stream); + console.log('upload', res); task = await cloudConvert.tasks.wait(task.id); + console.log('task', task); assert.equal(task.status, 'finished'); assert.equal(task.result.files[0].filename, 'input.png'); - await cloudConvert.tasks.delete(task.id); + const del = await cloudConvert.tasks.delete(task.id); + console.log(del); }).timeout(30000); }); }); From f2d3a42b27e43bbb7957a00dc88c2f272754d427 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sun, 20 Apr 2025 09:31:35 +0200 Subject: [PATCH 23/35] refactor: remove bad return types --- lib/JobsResource.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/JobsResource.ts b/lib/JobsResource.ts index 09d8953..2eec40a 100644 --- a/lib/JobsResource.ts +++ b/lib/JobsResource.ts @@ -76,7 +76,7 @@ export default class JobsResource { id: string, event: string, callback: (event: JobEventData) => void - ): Promise { + ) { this.cloudConvert.subscribe( `private-job.${id}`, `job.${event}`, @@ -84,11 +84,11 @@ export default class JobsResource { ); } - async subscribeTaskEvent( + subscribeTaskEvent( id: string, event: string, callback: (event: TaskEventData) => void - ): Promise { + ) { this.cloudConvert.subscribe( `private-job.${id}.tasks`, `task.${event}`, From b8b5cfa15b9b9c2aefeed624c1a850fc6660af35 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sun, 20 Apr 2025 09:31:57 +0200 Subject: [PATCH 24/35] fix: support S3 uploads in client --- lib/CloudConvert.ts | 53 +++++++++++++++++++++++++++++--------------- lib/TasksResource.ts | 3 ++- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 5575ff7..4cee4f6 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,5 +1,6 @@ -import { io, type Socket } from 'socket.io-client'; +import path from 'node:path'; import { Readable } from 'node:stream'; +import { io, type Socket } from 'socket.io-client'; import { version } from '../package.json'; import JobsResource, { type JobEventData } from './JobsResource'; import SignedUrlResource from './SignedUrlResource'; @@ -17,23 +18,26 @@ export type UploadFileSource = | AsyncIterable | NodeJS.ReadableStream; +function guessFilename(source: UploadFileSource): string | undefined { + return 'path' in source && typeof source.path === 'string' + ? path.basename(source.path) + : undefined; +} + export class UploadFile { private readonly attributes: Array<[key: string, value: unknown]> = []; private readonly data: AsyncIterable; constructor( data: UploadFileSource, - private readonly filename?: string + private readonly filename = guessFilename(data) ) { this.data = UploadFile.unifySources(data); } add(key: string, value: unknown) { this.attributes.push([key, value]); } - async *stream() { + async *stream(boundary: string) { const enc = new TextEncoder(); - const boundary = `----------${Array.from(Array(32)) - .map(() => Math.random().toString(36)[2] || 0) - .join('')}`; // Start multipart/form-data protocol yield enc.encode(`--${boundary}\r\n`); // Send all attributes @@ -117,22 +121,32 @@ export default class CloudConvert { async call( method: 'GET' | 'POST' | 'DELETE', route: string, - parameters?: UploadFile | object + parameters?: UploadFile | object, + options?: { presigned?: boolean; flat?: boolean } ) { const baseURL = this.useSandbox ? 'https://api.sandbox.cloudconvert.com/v2/' : `https://${ this.region ? this.region + '.' : '' }api.cloudconvert.com/v2/`; - return await this.callWithBase(baseURL, method, route, parameters); + return await this.callWithBase( + baseURL, + method, + route, + parameters, + options + ); } async callWithBase( baseURL: string, method: 'GET' | 'POST' | 'DELETE', route: string, - parameters?: UploadFile | object + parameters?: UploadFile | object, + options?: { presigned?: boolean; flat?: boolean } ) { + const presigned = options?.presigned ?? false; + const flat = options?.flat ?? false; const url = new URL(route, baseURL); const { contentType, search, body } = prepareParameters( method, @@ -142,8 +156,8 @@ export default class CloudConvert { url.search = search; } const headers = { - Authorization: `Bearer ${this.apiKey}`, 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)`, + ...(!presigned ? { Authorization: `Bearer ${this.apiKey}` } : {}), ...(contentType ? { 'Content-Type': contentType } : {}) }; const res = await fetch(url, { @@ -154,20 +168,20 @@ export default class CloudConvert { duplex: 'half' }); if (!res.ok) { - console.error('SND:', url, method, headers); - console.error('RCV:', res, await res.text()); // @ts-expect-error cause not present in types yet throw new Error(res.statusText, { cause: res }); } if ( - res.headers.get('Content-Type')?.toLowerCase() !== - 'application/json' + !res.headers + .get('content-type') + ?.toLowerCase() + .includes('application/json') ) { return undefined; } - const { data } = await res.json(); - return data; + const json = await res.json(); + return flat ? json : json.data; } subscribe( @@ -231,9 +245,12 @@ function prepareParameters( } if (data instanceof UploadFile) { + const boundary = `----------${Array.from(Array(32)) + .map(() => Math.random().toString(36)[2] || 0) + .join('')}`; return { - contentType: 'multipart/form-data', - body: asyncIterableToReadableStream(data.stream()) + contentType: `multipart/form-data; boundary=${boundary}`, + body: asyncIterableToReadableStream(data.stream(boundary)) }; } diff --git a/lib/TasksResource.ts b/lib/TasksResource.ts index c94493c..e7bafb4 100644 --- a/lib/TasksResource.ts +++ b/lib/TasksResource.ts @@ -609,7 +609,8 @@ export default class TasksResource { return await this.cloudConvert.call( 'POST', task.result.form.url, - uploadFile + uploadFile, + { presigned: true, flat: true } ); } From 04de11b9bf01c7a86c28e098d4e78be33941ff76 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sun, 20 Apr 2025 09:32:06 +0200 Subject: [PATCH 25/35] test: remove temporary logging --- tests/integration/TasksResourceTest.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/TasksResourceTest.ts b/tests/integration/TasksResourceTest.ts index 9ee7fca..9570be8 100644 --- a/tests/integration/TasksResourceTest.ts +++ b/tests/integration/TasksResourceTest.ts @@ -16,22 +16,18 @@ describe('TasksResource', () => { name: 'upload-test' }); - console.log('task', task); const stream = fs.createReadStream( __dirname + '/../integration/files/input.png' ); - const res = await cloudConvert.tasks.upload(task, stream); - console.log('upload', res); + await cloudConvert.tasks.upload(task, stream); task = await cloudConvert.tasks.wait(task.id); - console.log('task', task); assert.equal(task.status, 'finished'); assert.equal(task.result.files[0].filename, 'input.png'); - const del = await cloudConvert.tasks.delete(task.id); - console.log(del); + await cloudConvert.tasks.delete(task.id); }).timeout(30000); }); }); From c4f9f07ca4d811f1954675f4ee6cf2613c900eb4 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sun, 20 Apr 2025 12:43:23 +0200 Subject: [PATCH 26/35] docs: prefer find over filter --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e2294df..9f71598 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ const job = await cloudConvert.jobs.create({ } }); -const uploadTask = job.tasks.filter(task => task.name === 'upload-my-file')[0]; +const uploadTask = job.tasks.find(task => task.name === 'upload-my-file'); const inputFile = fs.createReadStream('./file.pdf'); From e5e0b9901c7ff65a37d071952f628aef8326d526 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Sun, 20 Apr 2025 12:47:05 +0200 Subject: [PATCH 27/35] refactor: make import more specific --- lib/CloudConvert.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 4cee4f6..9d24017 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,4 +1,4 @@ -import path from 'node:path'; +import { basename } from 'node:path'; import { Readable } from 'node:stream'; import { io, type Socket } from 'socket.io-client'; import { version } from '../package.json'; @@ -20,7 +20,7 @@ export type UploadFileSource = function guessFilename(source: UploadFileSource): string | undefined { return 'path' in source && typeof source.path === 'string' - ? path.basename(source.path) + ? basename(source.path) : undefined; } From 6b7fed24072a0108831db05847423d4a3446a1a5 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 23 Apr 2025 10:16:47 +0200 Subject: [PATCH 28/35] feat: re-introduce fileSize parameter, set Content-Length --- lib/CloudConvert.ts | 112 +++++++++++++++++++++++++++---------------- lib/TasksResource.ts | 5 +- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index 9d24017..ccccc13 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -1,3 +1,4 @@ +import { statSync } from 'node:fs'; import { basename } from 'node:path'; import { Readable } from 'node:stream'; import { io, type Socket } from 'socket.io-client'; @@ -18,20 +19,78 @@ export type UploadFileSource = | AsyncIterable | NodeJS.ReadableStream; -function guessFilename(source: UploadFileSource): string | undefined { - return 'path' in source && typeof source.path === 'string' - ? basename(source.path) - : undefined; +async function* unifySources( + data: UploadFileSource +): AsyncIterable { + if (data instanceof Uint8Array) { + yield data; + return; + } + + if (data instanceof Blob) { + yield data.bytes(); + return; + } + + if (Symbol.iterator in data) { + yield* data; + return; + } + + if (Symbol.asyncIterator in data) { + for await (const chunk of data) { + if (typeof chunk === 'string') + throw new Error( + 'bad file data, received string but expected Uint8Array' + ); + yield chunk; + } + return; + } +} + +function guessNameAndSize( + source: UploadFileSource, + fileName?: string, + fileSize?: number +): { name: string; size: number } { + const path = + 'path' in source && typeof source.path === 'string' + ? source.path + : undefined; + const name = + fileName ?? (path !== undefined ? basename(path) : undefined) ?? 'file'; + const size = + fileSize ?? + (source instanceof Uint8Array ? source.byteLength : undefined) ?? + (source instanceof Blob ? source.size : undefined) ?? + (path !== undefined + ? statSync(path, { throwIfNoEntry: false })?.size + : undefined) ?? + (fileName !== undefined + ? statSync(fileName, { throwIfNoEntry: false })?.size + : undefined); + if (size === undefined) { + throw new Error( + 'Could not determine the number of bytes, specify it explicitly when calling `upload`' + ); + } + return { name, size }; } export class UploadFile { private readonly attributes: Array<[key: string, value: unknown]> = []; private readonly data: AsyncIterable; - constructor( - data: UploadFileSource, - private readonly filename = guessFilename(data) - ) { - this.data = UploadFile.unifySources(data); + private readonly filename?: string; + private readonly fileSize: number; + constructor(data: UploadFileSource, filename?: string, fileSize?: number) { + this.data = unifySources(data); + const { name, size } = guessNameAndSize(data, filename, fileSize); + this.filename = name; + this.fileSize = size; + } + byteCount() { + return this.fileSize; } add(key: string, value: unknown) { this.attributes.push([key, value]); @@ -60,36 +119,6 @@ export class UploadFile { // End multipart/form-data protocol yield enc.encode(`\r\n--${boundary}--\r\n`); } - - static async *unifySources( - data: UploadFileSource - ): AsyncIterable { - if (data instanceof Uint8Array) { - yield data; - return; - } - - if (data instanceof Blob) { - yield data.bytes(); - return; - } - - if (Symbol.iterator in data) { - yield* data; - return; - } - - if (Symbol.asyncIterator in data) { - for await (const chunk of data) { - if (typeof chunk === 'string') - throw new Error( - 'bad file data, received string but expected Uint8Array' - ); - yield chunk; - } - return; - } - } } export default class CloudConvert { @@ -148,7 +177,7 @@ export default class CloudConvert { const presigned = options?.presigned ?? false; const flat = options?.flat ?? false; const url = new URL(route, baseURL); - const { contentType, search, body } = prepareParameters( + const { contentLength, contentType, search, body } = prepareParameters( method, parameters ); @@ -158,6 +187,7 @@ export default class CloudConvert { const headers = { 'User-Agent': `cloudconvert-node/v${version} (https://github.com/cloudconvert/cloudconvert-node)`, ...(!presigned ? { Authorization: `Bearer ${this.apiKey}` } : {}), + ...(contentLength ? { 'Content-Length': contentLength } : {}), ...(contentType ? { 'Content-Type': contentType } : {}) }; const res = await fetch(url, { @@ -230,6 +260,7 @@ function prepareParameters( method: 'GET' | 'POST' | 'DELETE', data?: UploadFile | object ): { + contentLength?: string; contentType?: string; body?: string | ReadableStream; search?: string; @@ -249,6 +280,7 @@ function prepareParameters( .map(() => Math.random().toString(36)[2] || 0) .join('')}`; return { + contentLength: data.byteCount().toString(), contentType: `multipart/form-data; boundary=${boundary}`, body: asyncIterableToReadableStream(data.stream(boundary)) }; diff --git a/lib/TasksResource.ts b/lib/TasksResource.ts index e7bafb4..81deb30 100644 --- a/lib/TasksResource.ts +++ b/lib/TasksResource.ts @@ -591,7 +591,8 @@ export default class TasksResource { async upload( task: Task | JobTask, stream: UploadFileSource, - filename?: string + filename?: string, + fileSize?: number ): Promise { if (task.operation !== 'import/upload') { throw new Error('The task operation is not import/upload'); @@ -601,7 +602,7 @@ export default class TasksResource { throw new Error('The task is not ready for uploading'); } - const uploadFile = new UploadFile(stream, filename); + const uploadFile = new UploadFile(stream, filename, fileSize); for (const parameter in task.result.form.parameters) { uploadFile.add(parameter, task.result.form.parameters[parameter]); } From aa59cf027afe279bedd6dcea119e897a75949758 Mon Sep 17 00:00:00 2001 From: KnorpelSenf Date: Wed, 23 Apr 2025 11:31:19 +0200 Subject: [PATCH 29/35] fix: precompute total body size --- lib/CloudConvert.ts | 49 +++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index ccccc13..f7957e8 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -89,35 +89,53 @@ export class UploadFile { this.filename = name; this.fileSize = size; } - byteCount() { - return this.fileSize; - } add(key: string, value: unknown) { this.attributes.push([key, value]); } - async *stream(boundary: string) { + toMultiPart(boundary: string): { + size: number; + stream: AsyncIterable; + } { const enc = new TextEncoder(); + const prefix: Uint8Array[] = []; + const suffix: Uint8Array[] = []; + // Start multipart/form-data protocol - yield enc.encode(`--${boundary}\r\n`); + prefix.push(enc.encode(`--${boundary}\r\n`)); // Send all attributes const separator = enc.encode(`\r\n--${boundary}\r\n`); let first = true; for (const [key, value] of this.attributes) { if (value == null) continue; - if (!first) yield separator; - yield enc.encode( - `content-disposition:form-data;name="${key}"\r\n\r\n${value}` + if (!first) prefix.push(separator); + prefix.push( + enc.encode( + `content-disposition:form-data;name="${key}"\r\n\r\n${value}` + ) ); first = false; } // Send file - if (!first) yield separator; - yield enc.encode( - `content-disposition:form-data;name="file";filename=${this.filename}\r\ncontent-type:application/octet-stream\r\n\r\n` + if (!first) prefix.push(separator); + prefix.push( + enc.encode( + `content-disposition:form-data;name="file";filename=${this.filename}\r\ncontent-type:application/octet-stream\r\n\r\n` + ) ); - yield* this.data; + const data = this.data; // End multipart/form-data protocol - yield enc.encode(`\r\n--${boundary}--\r\n`); + suffix.push(enc.encode(`\r\n--${boundary}--\r\n`)); + + const size = + prefix.reduce((sum, arr) => sum + arr.byteLength, 0) + + this.fileSize + + suffix.reduce((sum, arr) => sum + arr.byteLength, 0); + async function* concat() { + yield* prefix; + yield* data; + yield* suffix; + } + return { size, stream: concat() }; } } @@ -279,10 +297,11 @@ function prepareParameters( const boundary = `----------${Array.from(Array(32)) .map(() => Math.random().toString(36)[2] || 0) .join('')}`; + const { size, stream } = data.toMultiPart(boundary); return { - contentLength: data.byteCount().toString(), + contentLength: size.toString(), contentType: `multipart/form-data; boundary=${boundary}`, - body: asyncIterableToReadableStream(data.stream(boundary)) + body: asyncIterableToReadableStream(stream) }; } From aad2deaefb73ee676562afb334f1b47dd9479e1a Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Wed, 23 Apr 2025 12:51:02 +0200 Subject: [PATCH 30/35] add mocharc.json --- .mocharc.json | 6 ++++++ eslint.config.mjs | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .mocharc.json diff --git a/.mocharc.json b/.mocharc.json new file mode 100644 index 0000000..2ffcc8b --- /dev/null +++ b/.mocharc.json @@ -0,0 +1,6 @@ +{ + "require": "tsx", + "spec": [ + "tests/**/*.ts" + ] +} \ No newline at end of file diff --git a/eslint.config.mjs b/eslint.config.mjs index 7949ffc..d6f306f 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -24,7 +24,8 @@ export default defineConfig([ 'tests/unit/requests/', 'tests/unit/responses/', '.vscode/', - 'tsconfig.json' + 'tsconfig.json', + '.mocharc.json' ]), { extends: compat.extends( From 181e262959be75b0208032c04b554d936847bdfa Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Wed, 23 Apr 2025 12:51:13 +0200 Subject: [PATCH 31/35] test with node v23 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 03c4ab2..abea45f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - node-version: [20, 22] + node-version: [20, 22, 23] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 8e3c4b04c16a201dd595ba77411c5723890311bd Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Wed, 23 Apr 2025 12:51:19 +0200 Subject: [PATCH 32/35] update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f71598..c176830 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cloudconvert-node -This is the official Node.js SDK v2 for the [CloudConvert](https://cloudconvert.com/api/v2) **API v2**. +This is the official Node.js SDK for the [CloudConvert](https://cloudconvert.com/api/v2) API v2. [![Node.js Run Tests](https://github.com/cloudconvert/cloudconvert-node/actions/workflows/run-tests.yml/badge.svg)](https://github.com/cloudconvert/cloudconvert-node/actions/workflows/run-tests.yml) [![npm](https://img.shields.io/npm/v/cloudconvert.svg)](https://www.npmjs.com/package/cloudconvert) @@ -93,6 +93,9 @@ const inputFile = fs.createReadStream('./file.pdf'); await cloudConvert.tasks.upload(uploadTask, inputFile, 'file.pdf'); ``` +> **Note on custom streams**: +The length of the stream needs to be known prior to uploading. The SDK tries to automatically detect the file size of file-based read streams. If you are using a custom stream, you might need to pass a `fileSize` as fourth parameter to the `upload()` method. + ## Websocket Events The node SDK can subscribe to events of the [CloudConvert socket.io API](https://cloudconvert.com/api/v2/socket#socket). From 9e299d7e40cb9df14237de5acbb5c83d5bb1d8a4 Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Wed, 23 Apr 2025 13:04:35 +0200 Subject: [PATCH 33/35] limit parallel tests to avoid rate limits --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index abea45f..1807e39 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -14,6 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: + max-parallel: 2 matrix: node-version: [20, 22, 23] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ From 02857a2c98fb5128d44b9759118c14a2b0ad8f32 Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Wed, 23 Apr 2025 13:13:03 +0200 Subject: [PATCH 34/35] fix race condition in tests --- tests/integration/JobsResourceTest.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/JobsResourceTest.ts b/tests/integration/JobsResourceTest.ts index 2a9eaaf..14c529d 100644 --- a/tests/integration/JobsResourceTest.ts +++ b/tests/integration/JobsResourceTest.ts @@ -89,7 +89,10 @@ describe('JobsResource', () => { __dirname + '/../integration/files/input.png' ); - cloudConvert.tasks.upload(uploadTask, stream); + setTimeout(() => { + // for testing, we need to slow down the upload. otherwise we might miss the event because the job finishes too fast + cloudConvert.tasks.upload(uploadTask, stream); + }, 1000); const event = await new Promise(resolve => { cloudConvert.jobs.subscribeEvent(job.id, 'finished', resolve); From a55e11ae2e4e1c2e43197689c3202fbee24ceb38 Mon Sep 17 00:00:00 2001 From: Josias Montag Date: Thu, 24 Apr 2025 21:53:11 +0200 Subject: [PATCH 35/35] upload(): do not read file size of provided filename might get size of different file and result in unexpected results --- lib/CloudConvert.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/CloudConvert.ts b/lib/CloudConvert.ts index f7957e8..51677ad 100644 --- a/lib/CloudConvert.ts +++ b/lib/CloudConvert.ts @@ -66,9 +66,6 @@ function guessNameAndSize( (source instanceof Blob ? source.size : undefined) ?? (path !== undefined ? statSync(path, { throwIfNoEntry: false })?.size - : undefined) ?? - (fileName !== undefined - ? statSync(fileName, { throwIfNoEntry: false })?.size : undefined); if (size === undefined) { throw new Error(