feat: Zod-based Configuration class for cleaner SDK extension

.github/workflows/docs.yml

@@ -19,7 +19,7 @@ jobs:
         steps:
             -   uses: actions/checkout@v6
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24

.github/workflows/publish-to-npm.yml

@@ -77,7 +77,7 @@ jobs:
             - name: Bump canary versions
               if: inputs.dist-tag == 'next'
               run: |
-                    yarn turbo copy --force -- --canary --preid=beta
+                    yarn turbo copy --force -- --canary=major --preid=beta
 
             - name: Commit changes
               if: inputs.dist-tag == 'next'

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
             matrix:
                 # We don't test on Windows as the tests are flaky
                 os: [ ubuntu-22.04 ]
-                node-version: [ 18, 20, 22, 24 ]
+                node-version: [ 22, 24 ]
 
         runs-on: ${{ matrix.os }}
 
@@ -95,7 +95,7 @@ jobs:
                     token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
                     fetch-depth: 0
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24
@@ -106,7 +106,7 @@ jobs:
                     corepack enable
                     corepack prepare yarn@stable --activate
 
-            -   name: Activate cache for Node.js 20
+            -   name: Activate cache for Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     cache: 'yarn'
@@ -189,7 +189,7 @@ jobs:
                     token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
                     fetch-depth: 0
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24
@@ -203,7 +203,7 @@ jobs:
                     corepack enable
                     corepack prepare yarn@stable --activate
 
-            -   name: Activate cache for Node.js 20
+            -   name: Activate cache for Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     cache: 'yarn'

.github/workflows/test-ci.yml

@@ -2,9 +2,9 @@ name: Check
 
 on:
     push:
-        branches: [ master, renovate/** ]
+        branches: [ master, v4, renovate/** ]
     pull_request:
-        branches: [ master ]
+        branches: [ master, v4 ]
 
 env:
     YARN_IGNORE_NODE: 1
@@ -23,7 +23,7 @@ jobs:
                 # tests on windows are extremely unstable
                 # os: [ ubuntu-22.04, windows-2019 ]
                 os: [ ubuntu-22.04 ]
-                node-version: [ 18, 20, 22, 24 ]
+                node-version: [ 22, 24 ]
 
         steps:
             -   name: Cancel Workflow Action
@@ -97,7 +97,7 @@ jobs:
             -   name: Checkout Source code
                 uses: actions/checkout@v6
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24
@@ -108,7 +108,7 @@ jobs:
                     corepack enable
                     corepack prepare yarn@stable --activate
 
-            -   name: Activate cache for Node.js 20
+            -   name: Activate cache for Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     cache: 'yarn'
@@ -142,7 +142,7 @@ jobs:
             -   name: Checkout repository
                 uses: actions/checkout@v6
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24
@@ -153,7 +153,7 @@ jobs:
                     corepack enable
                     corepack prepare yarn@stable --activate
 
-            -   name: Activate cache for Node.js 20
+            -   name: Activate cache for Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     cache: 'yarn'
@@ -178,7 +178,7 @@ jobs:
 
     release_next:
         name: Release @next
-        if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+        if: github.event_name == 'push' && contains(github.event.ref, 'v4') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
         needs: build_and_test
         runs-on: ubuntu-22.04
 
@@ -240,22 +240,22 @@ jobs:
                         "dist-tag": "next"
                       }
 
-            -   name: Collect versions for Docker images
-                id: versions
-                run: |
-                    crawlee=`node -p "require('./packages/crawlee/package.json').version"`
-                    echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT
-
-            -   name: Trigger Docker image builds
-                uses: peter-evans/repository-dispatch@v4
-                # Trigger next images only if we have something new pushed
-                if: steps.changed-packages.outputs.changed_packages != '0'
-                with:
-                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
-                    repository: apify/apify-actor-docker
-                    event-type: build-node-images
-                    client-payload: >
-                        {
-                          "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
-                          "release_tag": "beta"
-                        }
+#            -   name: Collect versions for Docker images
+#                id: versions
+#                run: |
+#                    crawlee=`node -p "require('./packages/crawlee/package.json').version"`
+#                    echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT
+#
+#            -   name: Trigger Docker image builds
+#                uses: peter-evans/repository-dispatch@v4
+#                # Trigger next images only if we have something new pushed
+#                if: steps.changed-packages.outputs.changed_packages != '0'
+#                with:
+#                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+#                    repository: apify/apify-actor-docker
+#                    event-type: build-node-images
+#                    client-payload: >
+#                        {
+#                          "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
+#                          "release_tag": "beta"
+#                        }

.github/workflows/test-e2e.yml

@@ -29,7 +29,7 @@ jobs:
             -   name: Checkout repository
                 uses: actions/checkout@v6
 
-            -   name: Use Node.js 20
+            -   name: Use Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     node-version: 24
@@ -40,7 +40,7 @@ jobs:
                     corepack enable
                     corepack prepare yarn@stable --activate
 
-            -   name: Activate cache for Node.js 20
+            -   name: Activate cache for Node.js 24
                 uses: actions/setup-node@v6
                 with:
                     cache: 'yarn'

docs/examples/file_download.ts

@@ -2,11 +2,11 @@ import { FileDownload } from 'crawlee';
 
 // Create a FileDownload - a custom crawler instance that will download files from URLs.
 const crawler = new FileDownload({
-    async requestHandler({ body, request, contentType, getKeyValueStore }) {
+    async requestHandler({ request, response, contentType, getKeyValueStore }) {
         const url = new URL(request.url);
         const kvs = await getKeyValueStore();
 
-        await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
+        await kvs.setValue(url.pathname.replace(/\//g, '_'), response.body, { contentType: contentType.type });
     },
 });
 

docs/examples/file_download_stream.ts

@@ -23,32 +23,27 @@ function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; t
 
 // Create a FileDownload - a custom crawler instance that will download files from URLs.
 const crawler = new FileDownload({
-    async streamHandler({ stream, request, log, getKeyValueStore }) {
+    async requestHandler({ response, request, log, getKeyValueStore }) {
         const url = new URL(request.url);
 
         log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);
 
-        await new Promise<void>((resolve, reject) => {
-            // With the 'response' event, we have received the headers of the response.
-            stream.on('response', async (response) => {
-                const kvs = await getKeyValueStore();
-                await kvs.setValue(
-                    url.pathname.replace(/\//g, '_'),
-                    pipeline(
-                        stream,
-                        createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),
-                        (error) => {
-                            if (error) reject(error);
-                        },
-                    ),
-                    { contentType: response.headers['content-type'] },
-                );
-
-                log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
-
-                resolve();
-            });
-        });
+        if (!response.body) return;
+
+        const kvs = await getKeyValueStore();
+        await kvs.setValue(
+            url.pathname.replace(/\//g, '_'),
+            pipeline(
+                response.body,
+                createProgressTracker({ url, log, totalBytes: Number(response.headers.get('content-length')) }),
+                (error) => {
+                    if (error) log.error(`Failed to download ${url}: ${error.message}`);
+                },
+            ),
+            response.headers.get('content-type') ? { contentType: response.headers.get('content-type')! } : {},
+        );
+
+        log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
     },
 });
 

docs/examples/skip-navigation.ts

@@ -8,10 +8,15 @@ const crawler = new PlaywrightCrawler({
         // The request should have the navigation skipped
         if (request.skipNavigation) {
             // Request the image and get its buffer back
-            const imageResponse = await sendRequest({ responseType: 'buffer' });
-
-            // Save the image in the key-value store
-            await imageStore.setValue(`${request.userData.key}.png`, imageResponse.body);
+            const imageResponse = await sendRequest();
+
+            // Saves the image in the key-value store.
+            //
+            // Note: For large-scale file downloads, consider using FileDownload crawler:
+            // https://crawlee.dev/js/api/http-crawler/class/FileDownload
+            await imageStore.setValue(`${request.userData.key}.svg`, await imageResponse.bytes(), {
+                contentType: 'image/svg+xml',
+            });
 
             // Prevent executing the rest of the code as we do not need it
             return;

docs/guides/configuration.mdx

@@ -94,7 +94,6 @@ Storage directories are purged by default. If set to `false` - local storage dir
 
 #### `CRAWLEE_CONTAINERIZED`
 
-This variable is only effective when the systemInfoV2 experiment is enabled.
 Changes how crawlee measures its CPU and Memory usage and limits. If unset, crawlee will determine if it is containerised using common features of containerized environments using the `isContainerized` utility function.
  - A file at `/.dockerenv`.
  - A file at `/proc/self/cgroup` containing `docker`.