diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d77fb37c0c3..e3d2ffe29e10 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.yml index 82eb6d750550..d94000a01435 100644 --- a/.github/workflows/publish-to-npm.yml +++ b/.github/workflows/publish-to-npm.yml @@ -77,7 +77,7 @@ jobs: - name: Bump canary versions if: inputs.dist-tag == 'next' run: | - yarn turbo copy --force -- --canary --preid=beta + yarn turbo copy --force -- --canary=major --preid=beta - name: Commit changes if: inputs.dist-tag == 'next' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a151fd392dba..d250e7e86224 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: matrix: # We don't test on Windows as the tests are flaky os: [ ubuntu-22.04 ] - node-version: [ 18, 20, 22, 24 ] + node-version: [ 22, 24 ] runs-on: ${{ matrix.os }} @@ -95,7 +95,7 @@ jobs: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} fetch-depth: 0 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 @@ -106,7 +106,7 @@ jobs: corepack enable corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 + - name: Activate cache for Node.js 24 uses: actions/setup-node@v6 with: cache: 'yarn' @@ -189,7 +189,7 @@ jobs: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} fetch-depth: 0 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 @@ -203,7 +203,7 @@ jobs: corepack enable corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 + - name: Activate cache for Node.js 24 uses: actions/setup-node@v6 with: cache: 'yarn' diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index d5475794d64b..a7753e4445e0 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -2,9 +2,9 @@ name: Check on: push: - branches: [ master, renovate/** ] + branches: [ master, v4, renovate/** ] pull_request: - branches: [ master ] + branches: [ master, v4 ] env: YARN_IGNORE_NODE: 1 @@ -23,7 +23,7 @@ jobs: # tests on windows are extremely unstable # os: [ ubuntu-22.04, windows-2019 ] os: [ ubuntu-22.04 ] - node-version: [ 18, 20, 22, 24 ] + node-version: [ 22, 24 ] steps: - name: Cancel Workflow Action @@ -97,7 +97,7 @@ jobs: - name: Checkout Source code uses: actions/checkout@v6 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 @@ -108,7 +108,7 @@ jobs: corepack enable corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 + - name: Activate cache for Node.js 24 uses: actions/setup-node@v6 with: cache: 'yarn' @@ -142,7 +142,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 @@ -153,7 +153,7 @@ jobs: corepack enable corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 + - name: Activate cache for Node.js 24 uses: actions/setup-node@v6 with: cache: 'yarn' @@ -178,7 +178,7 @@ jobs: release_next: name: Release @next - if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')) + if: github.event_name == 'push' && contains(github.event.ref, 'v4') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')) needs: build_and_test runs-on: ubuntu-22.04 @@ -240,22 +240,22 @@ jobs: "dist-tag": "next" } - - name: Collect versions for Docker images - id: versions - run: | - crawlee=`node -p "require('./packages/crawlee/package.json').version"` - echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT - - - name: Trigger Docker image builds - uses: peter-evans/repository-dispatch@v4 - # Trigger next images only if we have something new pushed - if: steps.changed-packages.outputs.changed_packages != '0' - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - repository: apify/apify-actor-docker - event-type: build-node-images - client-payload: > - { - "crawlee_version": "${{ steps.versions.outputs.crawlee }}", - "release_tag": "beta" - } +# - name: Collect versions for Docker images +# id: versions +# run: | +# crawlee=`node -p "require('./packages/crawlee/package.json').version"` +# echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT +# +# - name: Trigger Docker image builds +# uses: peter-evans/repository-dispatch@v4 +# # Trigger next images only if we have something new pushed +# if: steps.changed-packages.outputs.changed_packages != '0' +# with: +# token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} +# repository: apify/apify-actor-docker +# event-type: build-node-images +# client-payload: > +# { +# "crawlee_version": "${{ steps.versions.outputs.crawlee }}", +# "release_tag": "beta" +# } diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 305a17b630bc..21f87db15f64 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -29,7 +29,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Use Node.js 20 + - name: Use Node.js 24 uses: actions/setup-node@v6 with: node-version: 24 @@ -40,7 +40,7 @@ jobs: corepack enable corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 + - name: Activate cache for Node.js 24 uses: actions/setup-node@v6 with: cache: 'yarn' diff --git a/docs/examples/file_download.ts b/docs/examples/file_download.ts index a6b42555e9ba..4ec682ea7002 100644 --- a/docs/examples/file_download.ts +++ b/docs/examples/file_download.ts @@ -2,11 +2,11 @@ import { FileDownload } from 'crawlee'; // Create a FileDownload - a custom crawler instance that will download files from URLs. const crawler = new FileDownload({ - async requestHandler({ body, request, contentType, getKeyValueStore }) { + async requestHandler({ request, response, contentType, getKeyValueStore }) { const url = new URL(request.url); const kvs = await getKeyValueStore(); - await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type }); + await kvs.setValue(url.pathname.replace(/\//g, '_'), response.body, { contentType: contentType.type }); }, }); diff --git a/docs/examples/file_download_stream.ts b/docs/examples/file_download_stream.ts index a7f39a70f59a..8931ecc2c764 100644 --- a/docs/examples/file_download_stream.ts +++ b/docs/examples/file_download_stream.ts @@ -23,32 +23,27 @@ function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; t // Create a FileDownload - a custom crawler instance that will download files from URLs. const crawler = new FileDownload({ - async streamHandler({ stream, request, log, getKeyValueStore }) { + async requestHandler({ response, request, log, getKeyValueStore }) { const url = new URL(request.url); log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`); - await new Promise((resolve, reject) => { - // With the 'response' event, we have received the headers of the response. - stream.on('response', async (response) => { - const kvs = await getKeyValueStore(); - await kvs.setValue( - url.pathname.replace(/\//g, '_'), - pipeline( - stream, - createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }), - (error) => { - if (error) reject(error); - }, - ), - { contentType: response.headers['content-type'] }, - ); - - log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`); - - resolve(); - }); - }); + if (!response.body) return; + + const kvs = await getKeyValueStore(); + await kvs.setValue( + url.pathname.replace(/\//g, '_'), + pipeline( + response.body, + createProgressTracker({ url, log, totalBytes: Number(response.headers.get('content-length')) }), + (error) => { + if (error) log.error(`Failed to download ${url}: ${error.message}`); + }, + ), + response.headers.get('content-type') ? { contentType: response.headers.get('content-type')! } : {}, + ); + + log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`); }, }); diff --git a/docs/examples/skip-navigation.ts b/docs/examples/skip-navigation.ts index 0bbde53c1375..679150e42705 100644 --- a/docs/examples/skip-navigation.ts +++ b/docs/examples/skip-navigation.ts @@ -8,10 +8,15 @@ const crawler = new PlaywrightCrawler({ // The request should have the navigation skipped if (request.skipNavigation) { // Request the image and get its buffer back - const imageResponse = await sendRequest({ responseType: 'buffer' }); - - // Save the image in the key-value store - await imageStore.setValue(`${request.userData.key}.png`, imageResponse.body); + const imageResponse = await sendRequest(); + + // Saves the image in the key-value store. + // + // Note: For large-scale file downloads, consider using FileDownload crawler: + // https://crawlee.dev/js/api/http-crawler/class/FileDownload + await imageStore.setValue(`${request.userData.key}.svg`, await imageResponse.bytes(), { + contentType: 'image/svg+xml', + }); // Prevent executing the rest of the code as we do not need it return; diff --git a/docs/experiments/systemInfoV2.mdx b/docs/experiments/systemInfoV2.mdx deleted file mode 100644 index 93f8f27e1afe..000000000000 --- a/docs/experiments/systemInfoV2.mdx +++ /dev/null @@ -1,95 +0,0 @@ ---- -id: experiments-system-infomation-v2 -title: System Infomation V2 -description: Improved autoscaling through cgroup aware metric collection. ---- - -import ApiLink from '@site/src/components/ApiLink'; - -:::caution - -This is an experimental feature. While we welcome testers, keep in mind that it is currently not recommended to use this in production. - -The API is subject to change, and we might introduce breaking changes in the future. - -Should you be using this, feel free to open issues on our [GitHub repository](https://github.com/apify/crawlee), and we'll take a look. - -::: - -Starting with the newest `crawlee` beta, we have introduced a new crawler option that enables an improved metric collection system. -This new system should collect cpu and memory metrics more accurately in containerised environments by checking for cgroup enforce limits. - -## How to enable the experiment - -:::note - -This example shows how to enable the experiment in the `CheerioCrawler`, -but you can apply this to any crawler type. - -::: - -```ts -import { CheerioCrawler, Configuration } from 'crawlee'; - -Configuration.set('systemInfoV2', true); - -const crawler = new CheerioCrawler({ - async requestHandler({ $, request }) { - const title = $('title').text(); - console.log(`The title of "${request.url}" is: ${title}.`); - }, -}); - -await crawler.run(['https://crawlee.dev']); -``` - -## Other changes - -:::info - -This section is only useful if you're a tinkerer and want to see what's going on under the hood. - -::: - -The existing solution checked the bare metal metrics for how much cpu and memory was being used and how much headroom was available. -This is an intuitive solution but unfortunately doesnt account for when there is an external limit on the amount of resources a process can consume. -This is often the case in containerized environments where each container will have a quota for its cpu and memory usage. - -This experiment attempts to address this issue by introducing a new `isContainerized()` utility function and changing the way resources are collected -when a container is detected. - -:::note - -This `isContainerized()` function is very similar to the existing `isDocker()` function however for now they both work side by side. -If this experiment is successful, eventualy `isDocker()` may eventually be depreciated in favour of `isContainerized()`. - -::: - -### Cgroup detection - -On linux, to detect if cgroup is available, we check if there is a directory at `/sys/fs/cgroup`. -If the directory exists, a version of cgroup is installed. -Next we check the version of cgroup installed by checking for a directory at `/sys/fs/cgroup/memory/`. -If it exists, cgroup V1 is installed. If it is missing, it is assumed cgroup V2 is installed. - -### CPU metric collection - -The existing solution worked by checking the fraction of cpu idle ticks to the total number of cpu ticks since the last profile. -If 100000 ticks elapse and 5000 were idle, the cpu is at 95% utilisation. - -In this experiment, the method of cpu load calculation depends on the result of `isContainerized()` or if set, the `CRAWLEE_CONTAINERIZED` environment variable. -If `isContainerized()` returns true, the new cgroup aware metric collection will be used over the "bare metal" numbers. -This works by inspecting the `/sys/fs/cgroup/cpuacct/cpuacct.usage`, `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` and `/sys/fs/cgroup/cpu/cpu.cfs_period_us` -files for cgroup V1 and the `/sys/fs/cgroup/cpu.stat` and `/sys/fs/cgroup/cpu.max` files for cgroup V2. -The actual cpu usage figure is calculated in the same manner as the "bare metal" figure by comparing the total number of ticks elapsed to the number -of idle ticks between profiles but by using the figures from the cgroup files. -If no cgroup quota is enforced, the "bare metal" numbers will be used. - -### Memory metric collection - -The existing solution was already cgroup aware however an improvement has been made to memory metric collection when running on windows. -The existing solution used an external package `apify/ps-tree` to find the amount of memory crawlee and any child processes were using. -On Windows, this package used the depreciated "WMIC" command line utility to determine memory usage. - -In this experiment, `apify/ps-tree` has been removed and replaced by the `packages/utils/src/internals/ps-tree.ts` file. This works in much the -same manner however, instead of using "WMIC", it uses "powershell" to collect the same data. \ No newline at end of file diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index 597c3dcc2fa4..22ce4ec2e583 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -94,7 +94,6 @@ Storage directories are purged by default. If set to `false` - local storage dir #### `CRAWLEE_CONTAINERIZED` -This variable is only effective when the systemInfoV2 experiment is enabled. Changes how crawlee measures its CPU and Memory usage and limits. If unset, crawlee will determine if it is containerised using common features of containerized environments using the `isContainerized` utility function. - A file at `/.dockerenv`. - A file at `/proc/self/cgroup` containing `docker`. diff --git a/docs/guides/custom-http-client/custom-http-client.mdx b/docs/guides/custom-http-client/custom-http-client.mdx index c593ec3ba239..9eb0918dbb7b 100644 --- a/docs/guides/custom-http-client/custom-http-client.mdx +++ b/docs/guides/custom-http-client/custom-http-client.mdx @@ -10,14 +10,34 @@ import CodeBlock from '@theme/CodeBlock'; import ImplementationSource from '!!raw-loader!./implementation.ts'; import UsageSource from '!!raw-loader!./usage.ts'; -The `BasicCrawler` class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate` or `axios`. +The `BasicCrawler` class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate`. -The HTTP client implementation needs to conform to the `BaseHttpClient` interface. For a rough idea on how it might look, see a skeleton implementation that uses the standard `fetch` interface: +## Built-in HTTP clients + +Crawlee provides several HTTP client implementations out of the box: + +- **`GotScrapingHttpClient`** (default) - Uses the `got-scraping` library for browser-like requests with support for custom headers, browser fingerprints, and proxies. +- **`ImpitHttpClient`** - Uses the `impit` library for making requests that closely mimic browser behavior. +- **`FetchHttpClient`** - Simple implementation using the native `fetch` API (does not support proxies). + +## Implementing a custom HTTP client + +To create a custom HTTP client, extend the `BaseHttpClient` abstract class from `@crawlee/http-client`. The base class handles common functionality like cookie management, redirect following, session integration, proxy support, and timeout handling. + +Your custom implementation only needs to override the `fetch` method to perform the actual network request: {ImplementationSource} +By extending `BaseHttpClient`, your implementation automatically gets: +- Cookie jar management (applying cookies before requests, saving cookies from responses) +- Automatic redirect following (up to 10 redirects) +- Session integration (proxy URL and cookies from session) +- Timeout handling via AbortSignal +- Proxy URL support + You may then instantiate it and pass to a crawler constructor: {UsageSource} -Please note that the interface is experimental and it will likely change with Crawlee version 4. +Alternatively, you can implement the `BaseHttpClient` interface directly if you need full control over all aspects of the HTTP request handling, including cookies, redirects, and sessions. However, this approach requires implementing significantly more logic yourself. + diff --git a/docs/guides/custom-http-client/implementation.ts b/docs/guides/custom-http-client/implementation.ts index 504f0b532f98..aac71784ff7e 100644 --- a/docs/guides/custom-http-client/implementation.ts +++ b/docs/guides/custom-http-client/implementation.ts @@ -1,122 +1,14 @@ -import type { - BaseHttpClient, - HttpRequest, - HttpResponse, - RedirectHandler, - ResponseTypes, - StreamingHttpResponse, -} from '@crawlee/core'; -import { Readable } from 'node:stream'; - -export class CustomHttpClient implements BaseHttpClient { - async sendRequest( - request: HttpRequest, - ): Promise> { - const requestHeaders = new Headers(); - for (let [headerName, headerValues] of Object.entries(request.headers ?? {})) { - if (headerValues === undefined) { - continue; - } - - if (!Array.isArray(headerValues)) { - headerValues = [headerValues]; - } - - for (const value of headerValues) { - requestHeaders.append(headerName, value); - } - } - - const response = await fetch(request.url, { - method: request.method, - headers: requestHeaders, - body: request.body as string, // TODO implement stream/generator handling - signal: request.signal, - // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...) - }); - - const headers: Record = {}; - - response.headers.forEach((value, headerName) => { - headers[headerName] = value; - }); - - return { - complete: true, - request, - url: response.url, - statusCode: response.status, - redirectUrls: [], // TODO you need to handle redirects manually to track them - headers, - trailers: {}, // TODO not supported by fetch - ip: undefined, - body: - request.responseType === 'text' - ? await response.text() - : request.responseType === 'json' - ? await response.json() - : Buffer.from(await response.text()), - }; - } - - async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise { - const fetchResponse = await fetch(request.url, { - method: request.method, - headers: new Headers(), - body: request.body as string, // TODO implement stream/generator handling - signal: request.signal, - // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...) - }); - - const headers: Record = {}; // TODO same as in sendRequest() - - async function* read() { - const reader = fetchResponse.body?.getReader(); - - const stream = new ReadableStream({ - start(controller) { - if (!reader) { - return null; - } - return pump(); - function pump(): Promise { - return reader!.read().then(({ done, value }) => { - // When no more data needs to be consumed, close the stream - if (done) { - controller.close(); - return; - } - // Enqueue the next data chunk into our target stream - controller.enqueue(value); - return pump(); - }); - } - }, - }); - - for await (const chunk of stream) { - yield chunk; - } - } - - const response = { - complete: false, - request, - url: fetchResponse.url, - statusCode: fetchResponse.status, - redirectUrls: [], // TODO you need to handle redirects manually to track them - headers, - trailers: {}, // TODO not supported by fetch - ip: undefined, - stream: Readable.from(read()), - get downloadProgress() { - return { percent: 0, transferred: 0 }; // TODO track this - }, - get uploadProgress() { - return { percent: 0, transferred: 0 }; // TODO track this - }, - }; - - return response; +import { BaseHttpClient, type CustomFetchOptions } from '@crawlee/http-client'; + +/** + * A simple HTTP client implementation using the native `fetch` API. + * + * Custom implementations only need to override the `fetch` method. + */ +export class CustomFetchClient extends BaseHttpClient { + protected override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise { + // The base class handles cookies, redirects, sessions, and timeouts. + // We only need to perform the actual network request here. + return fetch(request, options); } } diff --git a/docs/guides/custom-http-client/usage.ts b/docs/guides/custom-http-client/usage.ts index ebe52c236d3b..28fa63c5802a 100644 --- a/docs/guides/custom-http-client/usage.ts +++ b/docs/guides/custom-http-client/usage.ts @@ -1,8 +1,8 @@ import { HttpCrawler } from 'crawlee'; -import { CustomHttpClient } from './implementation.js'; +import { CustomFetchClient } from './implementation.js'; const crawler = new HttpCrawler({ - httpClient: new CustomHttpClient(), + httpClient: new CustomFetchClient(), async requestHandler() { /* ... */ }, diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdx index 8bf385f1c5b5..70dce2f72f9b 100644 --- a/docs/guides/proxy_management.mdx +++ b/docs/guides/proxy_management.mdx @@ -31,7 +31,7 @@ import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspectio and most effective ways of preventing access to a website. It is therefore paramount for a good web scraping library to provide easy to use but powerful tools which can work around IP blocking. The most powerful weapon in our anti IP blocking arsenal is a -[proxy server](https://en.wikipedia.org/wiki/Proxy_server). +[proxy server](https://en.wikipedia.org/wiki/Proxy_server). With Crawlee we can use our own proxy servers or proxy servers acquired from third-party providers. @@ -105,7 +105,7 @@ You can also provide a list of proxy tiers to the `ProxyConfiguration` class. Th :::warning -Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)). +Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)). Using this configuration through the `newUrl` calls will not yield the expected results. @@ -162,9 +162,7 @@ Our crawlers will now use the selected proxies for all connections. ## IP Rotation and session management -​`proxyConfiguration.newUrl()` allows us to pass a `sessionId` parameter. It will then be used to create a `sessionId`-`proxyUrl` pair, and subsequent `newUrl()` calls with the same `sessionId` will always return the same `proxyUrl`. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and `SessionPool` class for more information on how keeping a real session helps us avoid blocking. - -When no `sessionId` is provided, our proxy URLs are rotated round-robin. +Each call to `proxyConfiguration.newUrl()` generates a new proxy URL. Crawler instances pair these URLs with `Session` instances and rotate those together with browser fingerprints, impersonated headers, and more. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and `SessionPool` class for more information on how keeping a real session helps us avoid blocking. @@ -202,7 +200,7 @@ When no `sessionId` is provided, our proxy URLs are rotated round-robin. ## Inspecting current proxy in Crawlers `HttpCrawler`, `CheerioCrawler`, `JSDOMCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxy -in their `requestHandler` using a `proxyInfo` object. +in their `requestHandler` using a `proxyInfo` object. With the `proxyInfo` object, we can easily access the proxy URL. diff --git a/docs/guides/proxy_management_session_standalone.ts b/docs/guides/proxy_management_session_standalone.ts index bc2010f79b18..dec095d03408 100644 --- a/docs/guides/proxy_management_session_standalone.ts +++ b/docs/guides/proxy_management_session_standalone.ts @@ -4,10 +4,4 @@ const proxyConfiguration = new ProxyConfiguration({ /* opts */ }); -const sessionPool = await SessionPool.open({ - /* opts */ -}); - -const session = await sessionPool.getSession(); - -const proxyUrl = await proxyConfiguration.newUrl(session.id); +const proxyUrl = await proxyConfiguration.newUrl(); diff --git a/docs/guides/request_storage_queue_crawler.ts b/docs/guides/request_storage_queue_crawler.ts index 07af11ffa712..d9c37f57f7de 100644 --- a/docs/guides/request_storage_queue_crawler.ts +++ b/docs/guides/request_storage_queue_crawler.ts @@ -4,7 +4,7 @@ import { CheerioCrawler } from 'crawlee'; // It's used the same way for Puppeteer/Playwright crawlers. const crawler = new CheerioCrawler({ // Note that we're not specifying the requestQueue here - async requestHandler({ crawler, enqueueLinks }) { + async requestHandler({ enqueueLinks }) { // Add new request to the queue await crawler.addRequests([{ url: 'https://example.com/new-page' }]); // Add links found on page to the queue diff --git a/docs/guides/request_storage_queue_list.ts b/docs/guides/request_storage_queue_list.ts index 456caf1c015c..37de17eecdf4 100644 --- a/docs/guides/request_storage_queue_list.ts +++ b/docs/guides/request_storage_queue_list.ts @@ -25,7 +25,7 @@ const crawler = new PuppeteerCrawler({ requestQueue, // Each request from the request list is enqueued to the request queue one by one. // At this point request with the same URL would exist in the list and the queue - async requestHandler({ crawler, enqueueLinks }) { + async requestHandler({ enqueueLinks }) { // Add new request to the queue await crawler.addRequests(['http://www.example.com/new-page']); diff --git a/docs/guides/request_storage_queue_only.ts b/docs/guides/request_storage_queue_only.ts index 5d9a31379597..3054135504f3 100644 --- a/docs/guides/request_storage_queue_only.ts +++ b/docs/guides/request_storage_queue_only.ts @@ -15,7 +15,7 @@ const sources = [ // The crawler will automatically process requests from the queue. // It's used the same way for Cheerio/Playwright crawlers const crawler = new PuppeteerCrawler({ - async requestHandler({ crawler, enqueueLinks }) { + async requestHandler({ enqueueLinks }) { // Add new request to the queue await crawler.addRequests(['http://www.example.com/new-page']); diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts index c7b7ec37c361..38b65ee9d31d 100644 --- a/docs/guides/session_management_basic.ts +++ b/docs/guides/session_management_basic.ts @@ -1,5 +1,6 @@ import { BasicCrawler, ProxyConfiguration } from 'crawlee'; -import { gotScraping } from 'got-scraping'; +import { Impit } from 'impit'; +import { Cookie } from 'tough-cookie'; const proxyConfiguration = new ProxyConfiguration({ /* opts */ @@ -12,22 +13,19 @@ const crawler = new BasicCrawler({ sessionPoolOptions: { maxPoolSize: 100 }, async requestHandler({ request, session }) { const { url } = request; - const requestOptions = { - url, - // We use session id in order to have the same proxyUrl - // for all the requests using the same session. - proxyUrl: await proxyConfiguration.newUrl(session?.id), - throwHttpErrors: false, + const client = new Impit({ + proxyUrl: await proxyConfiguration.newUrl(), + ignoreTlsErrors: true, headers: { // If you want to use the cookieJar. // This way you get the Cookie headers string from session. - Cookie: session?.getCookieString(url), + Cookie: session?.getCookieString(url) ?? '', }, - }; + }); let response; try { - response = await gotScraping(requestOptions); + response = await client.fetch(url); } catch (e) { if (e === 'SomeNetworkError') { // If a network error happens, such as timeout, socket hangup, etc. @@ -39,9 +37,9 @@ const crawler = new BasicCrawler({ } // Automatically retires the session based on response HTTP status code. - session?.retireOnBlockedStatusCodes(response.statusCode); + session?.retireOnBlockedStatusCodes(response.status); - if (response.body.includes('You are blocked!')) { + if ((await response.text()).includes('You are blocked!')) { // You are sure it is blocked. // This will throw away the session. session?.retire(); @@ -51,6 +49,17 @@ const crawler = new BasicCrawler({ // No need to call session.markGood -> BasicCrawler calls it for you. // If you want to use the CookieJar in session you need. - session?.setCookiesFromResponse(response); + if (response.headers.has('set-cookie')) { + const newCookies = response.headers + .get('set-cookie') + ?.split(';') + .map((x) => Cookie.parse(x)); + + newCookies?.forEach((cookie) => { + if (cookie) { + session?.cookieJar?.setCookie(cookie, url); + } + }); + } }, }); diff --git a/docs/package.json b/docs/package.json index 26a6039ff021..105322cf9d82 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,6 +10,7 @@ "typescript": "^5.9.3" }, "dependencies": { + "impit": "^0.7.1", "playwright-extra": "^4.3.6", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2" diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md new file mode 100644 index 000000000000..12a87d5db0e8 --- /dev/null +++ b/docs/upgrading/upgrading_v4.md @@ -0,0 +1,116 @@ +--- +id: upgrading-to-v4 +title: Upgrading to v4 +--- + +import ApiLink from '@site/src/components/ApiLink'; + +This page summarizes most of the breaking changes in Crawlee v4. + +## ECMAScript modules + +Crawlee v4 is a native ESM package now. It can be still consumed from a CJS project, as long as you use TypeScript and Node.js version that supports `require(esm)`. + +## Node 22+ required + +Support for older node versions was dropped. + +## TypeScript 5.8+ required + +Support for older TypeScript versions was dropped. Older versions might work too, but only if your project is also ESM. + +## Cheerio v1 + +Previously, we kept the dependency on cheerio locked to the latest RC version, since there were many breaking changes introduced in v1.0. This release bumps cheerio to the stable v1. Also, we now use the default `parse5` internally. + +## Deprecated crawler options are removed + +The crawler following options are removed: + +- `handleRequestFunction` -> `requestHandler` +- `handlePageFunction` -> `requestHandler` +- `handleRequestTimeoutSecs` -> `requestHandlerTimeoutSecs` +- `handleFailedRequestFunction` -> `failedRequestHandler` + +## Underscore prefix is removed from many protected and private methods + +- `BasicCrawler._runRequestHandler` -> `BasicCrawler.runRequestHandler` + +## Removed symbols + +- `BasicCrawler._cleanupContext` (protected) - this is now handled by the `ContextPipeline` +- `BasicCrawler.isRequestBlocked` (protected) +- `BrowserRequestHandler` and `BrowserErrorHandler` types in `@crawlee/browser` +- `BrowserCrawler.userProvidedRequestHandler` (protected) +- `BrowserCrawler.requestHandlerTimeoutInnerMillis` (protected) +- `BrowserCrawler._enhanceCrawlingContextWithPageInfo` (protected) +- `BrowserCrawler._handleNavigation` (protected) +- `HttpCrawler.userRequestHandlerTimeoutMillis` (protected) +- `HttpCrawler._handleNavigation` (protected) +- `HttpCrawler._parseHTML` (protected) +- `HttpCrawler._parseResponse` (protected) - made private +- `HttpCrawler.use` and the `CrawlerExtension` class (experimental) - the `ContextPipeline` should be used for extending the crawler +- `FileDownloadOptions.streamHandler` - streaming should now be handled directly in the `requestHandler` instead +- `playwrightUtils.registerUtilsToContext` and `puppeteerUtils.registerUtilsToContext` - this is now added to the context via `ContextPipeline` composition +- `puppeteerUtils.blockResources` and `puppeteerUtils.cacheResponses` (deprecated) + +### The protected `BasicCrawler.crawlingContexts` map is removed + +The property was not used by the library itself and re-implementing the functionality in user code is fairly straightforward. + +## Removed crawling context properties + +### Crawling context no longer includes Error for failed requests + +The crawling context no longer includes the `Error` object for failed requests. Use the second parameter of the `errorHandler` or `failedRequestHandler` callbacks to access the error. + +### Crawling context no longer includes a reference to the crawler itself + +This was previously accessible via `context.crawler`. If you want to restore the functionality, you may use the `extendContext` option of the crawler: + +```ts +const crawler = new CheerioCrawler({ + extendContext: () => ({ crawler }), + requestHandler: async (context) => { + if (Math.random() < 0.01) { + context.crawler.stop() + } + } +}) +``` + +## Crawling context is strictly typed + +Previously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context. + +## `additionalBlockedStatusCodes` parameter is removed + +`additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes` method is removed. Use the `blockedStatusCodes` crawler option instead. + +## Remove `experimentalContainers` option + +This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions. + +## Available resource detection + +In v3, we introduced a new way to detect available resources for the crawler, available via `systemInfoV2` flag. In v4, this is the default way to detect available resources. The old way is removed completely together with the `systemInfoV2` flag. + +## `HttpClient` instances return `Response` objects + +The interface of `HttpClient` instances was changed to return the [native `Response` objects](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of custom `HttpResponse` objects. + +## `CrawlingContext.response` is now of type `Response` + +The `CrawlingContext.response` property is now of type [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of `HttpResponse`. `CrawlingContext.sendRequest` method now returns `Response` objects as well. + +## Crawling context in the `FileDownload` crawler no longer includes `body` and `stream` properties + +The crawling context in the `FileDownload` crawler no longer includes the `body` and `stream` properties. These can be accessed directly via the `response` property instead, e.g. `context.response.bytes()` or `context.response.body`. + +## `KeyValueStore.getPublicUrl` is now async + +The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public URL directly from the storage client. + +## `preNavigationHooks` in `HttpCrawler` no longer accepts `gotOptions` object + +The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead. diff --git a/docs/yarn.lock b/docs/yarn.lock index a0726729186d..ac9128687298 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -69,6 +69,7 @@ __metadata: version: 0.0.0-use.local resolution: "crawlee-docs@workspace:." dependencies: + impit: "npm:^0.7.1" playwright-extra: "npm:^4.3.6" puppeteer-extra: "npm:^3.3.6" puppeteer-extra-plugin-stealth: "npm:^2.11.2" @@ -157,6 +158,95 @@ __metadata: languageName: node linkType: hard +"impit-darwin-arm64@npm:0.7.1": + version: 0.7.1 + resolution: "impit-darwin-arm64@npm:0.7.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"impit-darwin-x64@npm:0.7.1": + version: 0.7.1 + resolution: "impit-darwin-x64@npm:0.7.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"impit-linux-arm64-gnu@npm:0.7.1": + version: 0.7.1 + resolution: "impit-linux-arm64-gnu@npm:0.7.1" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"impit-linux-arm64-musl@npm:0.7.1": + version: 0.7.1 + resolution: "impit-linux-arm64-musl@npm:0.7.1" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"impit-linux-x64-gnu@npm:0.7.1": + version: 0.7.1 + resolution: "impit-linux-x64-gnu@npm:0.7.1" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"impit-linux-x64-musl@npm:0.7.1": + version: 0.7.1 + resolution: "impit-linux-x64-musl@npm:0.7.1" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"impit-win32-arm64-msvc@npm:0.7.1": + version: 0.7.1 + resolution: "impit-win32-arm64-msvc@npm:0.7.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"impit-win32-x64-msvc@npm:0.7.1": + version: 0.7.1 + resolution: "impit-win32-x64-msvc@npm:0.7.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"impit@npm:^0.7.1": + version: 0.7.1 + resolution: "impit@npm:0.7.1" + dependencies: + impit-darwin-arm64: "npm:0.7.1" + impit-darwin-x64: "npm:0.7.1" + impit-linux-arm64-gnu: "npm:0.7.1" + impit-linux-arm64-musl: "npm:0.7.1" + impit-linux-x64-gnu: "npm:0.7.1" + impit-linux-x64-musl: "npm:0.7.1" + impit-win32-arm64-msvc: "npm:0.7.1" + impit-win32-x64-msvc: "npm:0.7.1" + dependenciesMeta: + impit-darwin-arm64: + optional: true + impit-darwin-x64: + optional: true + impit-linux-arm64-gnu: + optional: true + impit-linux-arm64-musl: + optional: true + impit-linux-x64-gnu: + optional: true + impit-linux-x64-musl: + optional: true + impit-win32-arm64-msvc: + optional: true + impit-win32-x64-msvc: + optional: true + checksum: 10c0/25032be7069d725273180c8f7de8c03a8572d786e196ebfaaa93e8fe591f85464ebd4bb766a1de59212ba6aef02e05e547dfcabf11e6e922cb1c58fc722edd2f + languageName: node + linkType: hard + "inflight@npm:^1.0.4": version: 1.0.6 resolution: "inflight@npm:1.0.6" diff --git a/eslint.config.mjs b/eslint.config.mjs index 2092da6a7312..1034e74cb54a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,5 +1,6 @@ import tsEslint from 'typescript-eslint'; import tsStylistic from '@stylistic/eslint-plugin-ts'; +import apifyJs from '@apify/eslint-config/js'; import apify from '@apify/eslint-config/ts'; import prettier from 'eslint-config-prettier'; @@ -78,4 +79,13 @@ export default [ 'no-undef': 'off', }, }, + // { + // files: ['test/**/*'], + // rules: { + // ...apifyJs.rules, + // '@typescript-eslint/no-floating-promises': 'off', + // 'no-console': 'off', + // 'no-undef': 'off', + // }, + // }, ]; diff --git a/package.json b/package.json index 69575f547504..da3f57cce41f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@crawlee/root", "private": true, + "type": "module", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "workspaces": [ "packages/*" @@ -43,76 +44,78 @@ "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 vitest run --silent", "tsc-check-tests": "tsc --noEmit --project test/tsconfig.json", "coverage": "vitest --coverage", - "publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish", + "publish:next": "lerna publish from-package --contents dist --dist-tag v4 --force-publish", "release:next": "yarn build && yarn publish:next", "publish:prod": "lerna publish from-package --contents dist --force-publish", "release:prod": "yarn build && yarn publish:prod", "release:pin-versions": "turbo run copy -- -- --pin-versions", "lint": "eslint \"packages/**/*.ts\" \"test/**/*.ts\"", - "lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix", + "lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.{ts,mjs}\" --fix", "format": "biome format --write .", "format:check": "biome format .", "prepare": "husky" }, "devDependencies": { "@apify/eslint-config": "^1.0.0", - "@apify/log": "^2.4.0", - "@apify/tsconfig": "^0.1.0", + "@apify/log": "^2.5.18", + "@apify/tsconfig": "^0.1.1", "@biomejs/biome": "^2.2.5", "@commitlint/config-conventional": "^20.0.0", "@playwright/browser-chromium": "1.56.1", "@playwright/browser-firefox": "1.56.1", "@playwright/browser-webkit": "1.56.1", "@stylistic/eslint-plugin-ts": "^4.2.0", - "@types/content-type": "^1.1.5", - "@types/deep-equal": "^1.0.1", - "@types/domhandler": "^2.4.2", - "@types/express": "^4.17.13", - "@types/fs-extra": "^11.0.0", - "@types/inquirer": "^8.2.1", - "@types/is-ci": "^3.0.1", + "@types/content-type": "^1.1.8", + "@types/deep-equal": "^1.0.4", + "@types/domhandler": "^3.1.0", + "@types/express": "^5.0.1", + "@types/fs-extra": "^11.0.4", + "@types/inquirer": "^9.0.8", + "@types/is-ci": "^3.0.4", "@types/lodash.isequal": "^4.5.8", - "@types/lodash.merge": "^4.6.7", - "@types/mime-types": "^2.1.1", + "@types/lodash.merge": "^4.6.9", + "@types/mime-types": "^2.1.4", "@types/node": "^24.0.0", - "@types/proper-lockfile": "^4.1.2", - "@types/ps-tree": "^1.1.2", - "@types/rimraf": "^4.0.0", - "@types/sax": "^1.0.0", - "@types/semver": "^7.3.12", - "@types/stream-json": "^1.7.2", - "@types/yargs": "^17.0.26", + "@types/proper-lockfile": "^4.1.4", + "@types/ps-tree": "^1.1.6", + "@types/rimraf": "^4.0.5", + "@types/sax": "^1.2.7", + "@types/semver": "^7.7.0", + "@types/stream-json": "^1.7.8", + "@types/whatwg-mimetype": "^3.0.2", + "@types/yargs": "^17.0.33", "@vitest/coverage-v8": "^4.0.1", "apify": "*", - "apify-node-curl-impersonate": "^1.0.15", + "apify-node-curl-impersonate": "^1.0.23", "basic-auth-parser": "^0.0.2", - "body-parser": "^2.0.0", + "body-parser": "^2.2.0", "camoufox-js": "^0.8.0", "commitlint": "^20.0.0", "cross-env": "^10.0.0", - "deep-equal": "^2.0.5", - "eslint": "^9.23.0", - "eslint-config-prettier": "^10.1.1", - "express": "^4.18.1", - "fs-extra": "^11.0.0", + "deep-equal": "^2.2.3", + "eslint": "^9.26.0", + "eslint-config-prettier": "^10.1.3", + "express": "^5.1.0", + "fs-extra": "^11.3.0", "gen-esm-wrapper": "^1.1.3", - "globals": "^16.0.0", + "globals": "^16.1.0", "globby": "^15.0.0", - "got": "^13.0.0", - "husky": "^9.0.11", - "is-ci": "^4.0.0", + "got": "^14.4.7", + "husky": "^9.1.7", + "is-ci": "^4.1.0", "lerna": "^9.0.0", "lint-staged": "^16.0.0", - "nock": "^13.4.0", + "nock": "^14.0.10", "playwright": "1.56.1", "portastic": "^1.0.1", - "proxy": "^1.0.2", + "proxy": "^2.2.0", "puppeteer": "24.28.0", - "rimraf": "^6.0.0", - "tsx": "^4.4.0", - "turbo": "^2.1.0", - "typescript": "^5.7.3", - "typescript-eslint": "^8.28.0", + "rimraf": "^6.0.1", + "tsx": "^4.19.4", + "turbo": "^2.5.3", + "typescript": "^5.8.3", + "typescript-eslint": "^8.32.0", + "vite-tsconfig-paths": "^5.1.4", "vitest": "^4.0.1" }, "packageManager": "yarn@4.10.3", diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.json index 1eca9b8ea536..053bf35ae644 100644 --- a/packages/basic-crawler/package.json +++ b/packages/basic-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/basic", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "author": { @@ -38,25 +32,25 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/log": "^2.4.0", - "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.7.10", - "@crawlee/core": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "csv-stringify": "^6.2.0", - "fs-extra": "^11.0.0", - "got-scraping": "^4.0.0", - "ow": "^0.28.1", - "tldts": "^7.0.0", - "tslib": "^2.4.0", - "type-fest": "^4.0.0" + "@apify/log": "^2.5.18", + "@apify/timeout": "^0.3.2", + "@apify/utilities": "^2.15.5", + "@crawlee/core": "4.0.0", + "@crawlee/got-scraping-client": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "csv-stringify": "^6.5.2", + "fs-extra": "^11.3.0", + "ow": "^2.0.0", + "tldts": "^7.0.6", + "tslib": "^2.8.1", + "type-fest": "^4.41.0" } } diff --git a/packages/basic-crawler/src/index.ts b/packages/basic-crawler/src/index.ts index ba211fc2b61e..ab98a100405a 100644 --- a/packages/basic-crawler/src/index.ts +++ b/packages/basic-crawler/src/index.ts @@ -1,4 +1,3 @@ export * from '@crawlee/core'; -export * from './internals/basic-crawler'; -export * from './internals/constants'; +export * from './internals/basic-crawler.js'; export { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils'; diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index a9eeb1461fda..c0255a7ae856 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -1,10 +1,10 @@ +import { writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, - BaseHttpClient, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, @@ -13,12 +13,10 @@ import type { GetUserDataFromRequest, IRequestList, IRequestManager, - LoadedContext, - ProxyInfo, + ProxyConfiguration, Request, RequestsLike, RequestTransform, - RestrictedCrawlingContext, RouterHandler, RouterRoutes, Session, @@ -31,16 +29,20 @@ import type { import { AutoscaledPool, Configuration, + ContextPipeline, + ContextPipelineCleanupError, + ContextPipelineInitializationError, + ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, EventType, - GotScrapingHttpClient, KeyValueStore, mergeCookies, NonRetryableError, purgeDefaultStorages, + RequestHandlerError, RequestListAdapter, RequestManagerTandem, RequestProvider, @@ -54,13 +56,21 @@ import { Statistics, validators, } from '@crawlee/core'; -import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types'; +import { GotScrapingHttpClient } from '@crawlee/got-scraping-client'; +import type { + Awaitable, + BaseHttpClient, + BatchAddRequestsResult, + Dictionary, + ProxyInfo, + SetStatusMessageOptions, +} from '@crawlee/types'; import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils'; import { stringify } from 'csv-stringify/sync'; -import { ensureDir, writeFile, writeJSON } from 'fs-extra'; -import ow, { ArgumentError } from 'ow'; +import { ensureDir, writeJSON } from 'fs-extra/esm'; +import ow from 'ow'; import { getDomain } from 'tldts'; -import type { SetRequired } from 'type-fest'; +import type { ReadonlyDeep, SetRequired } from 'type-fest'; import { LruCache } from '@apify/datastructures'; import type { Log } from '@apify/log'; @@ -68,35 +78,9 @@ import defaultLog, { LogLevel } from '@apify/log'; import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout'; import { cryptoRandomObjectId } from '@apify/utilities'; -import { createSendRequest } from './send-request'; +import { createSendRequest } from './send-request.js'; -export interface BasicCrawlingContext - extends CrawlingContext { - /** - * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue} - * currently used by the crawler. - * - * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions - * and override settings of the enqueued {@apilink Request} objects. - * - * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example - * for more details regarding its usage. - * - * **Example usage** - * - * ```ts - * async requestHandler({ enqueueLinks }) { - * await enqueueLinks({ - * urls: [...], - * }); - * }, - * ``` - * - * @param [options] All `enqueueLinks()` parameters are passed via an options object. - * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object. - */ - enqueueLinks(options?: SetRequired): Promise; -} +export interface BasicCrawlingContext extends CrawlingContext {} /** * Since there's no set number of seconds before the container is terminated after @@ -109,13 +93,12 @@ export interface BasicCrawlingContext */ const SAFE_MIGRATION_WAIT_MILLIS = 20000; -export type RequestHandler< - Context extends CrawlingContext = LoadedContext, -> = (inputs: LoadedContext) => Awaitable; +export type RequestHandler = (inputs: Context) => Awaitable; export type ErrorHandler< - Context extends CrawlingContext = LoadedContext, -> = (inputs: LoadedContext, error: Error) => Awaitable; + Context extends CrawlingContext = CrawlingContext, + ExtendedContext extends Context = Context, +> = (inputs: Context & Partial, error: Error) => Awaitable; export interface StatusMessageCallbackParams< Context extends CrawlingContext = BasicCrawlingContext, @@ -132,7 +115,18 @@ export type StatusMessageCallback< Crawler extends BasicCrawler = BasicCrawler, > = (params: StatusMessageCallbackParams) => Awaitable; -export interface BasicCrawlerOptions { +export type RequireContextPipeline< + DefaultContextType extends CrawlingContext, + FinalContextType extends DefaultContextType, +> = DefaultContextType extends FinalContextType + ? {} + : { contextPipelineBuilder: () => ContextPipeline }; + +export interface BasicCrawlerOptions< + Context extends CrawlingContext = CrawlingContext, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, +> { /** * User-provided function that performs the logic of the crawler. It is called for each URL to crawl. * @@ -150,29 +144,37 @@ export interface BasicCrawlerOptions>; + requestHandler?: RequestHandler; /** - * User-provided function that performs the logic of the crawler. It is called for each URL to crawl. + * Allows the user to extend the crawling context passed to the request handler with custom functionality. * - * The function receives the {@apilink BasicCrawlingContext} as an argument, - * where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl. + * **Example usage:** * - * The function must return a promise, which is then awaited by the crawler. + * ```javascript + * import { BasicCrawler } from 'crawlee'; * - * If the function throws an exception, the crawler will try to re-crawl the - * request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. - * If all the retries fail, the crawler calls the function - * provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter. - * To make this work, we should **always** - * let our function throw exceptions rather than catch them. - * The exceptions are logged to the request using the - * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function. + * // Create a crawler instance + * const crawler = new BasicCrawler({ + * extendContext(context) => ({ + * async customHelper() { + * await context.pushData({ url: context.request.url }) + * } + * }), + * async requestHandler(context) { + * await context.customHelper(); + * }, + * }); + * ``` + */ + extendContext?: (context: Context) => Awaitable; + + /** + * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter. * - * @deprecated `handleRequestFunction` has been renamed to `requestHandler` and will be removed in a future version. - * @ignore + * The option is not required if your crawler subclass does not extend the crawling context with custom information or helpers. */ - handleRequestFunction?: RequestHandler; + contextPipelineBuilder?: () => ContextPipeline; /** * Static list of URLs to be processed. @@ -204,14 +206,6 @@ export interface BasicCrawlerOptions; - - /** - * A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. - * - * The function receives the {@apilink BasicCrawlingContext} as the first argument, - * where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request. - * Second argument is the `Error` instance that - * represents the last error thrown during processing of the request. - */ - failedRequestHandler?: ErrorHandler; + errorHandler?: ErrorHandler; /** * A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. @@ -240,11 +224,8 @@ export interface BasicCrawlerOptions; + failedRequestHandler?: ErrorHandler; /** * Specifies the maximum number of retries allowed for a request if its processing fails. @@ -405,6 +386,25 @@ export interface BasicCrawlerOptions { +export class BasicCrawler< + Context extends CrawlingContext = CrawlingContext, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, +> { protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE'; + /** + * Tracks crawler instances that accessed shared state without having an explicit id. + * Used to detect and warn about multiple crawlers sharing the same state. + */ + private static useStateCrawlerIds = new Set(); + /** * A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests. */ @@ -528,19 +538,36 @@ export class BasicCrawler> = Router.create>(); + readonly router: RouterHandler = Router.create(); + + private contextPipelineBuilder: () => ContextPipeline; + private _contextPipeline?: ContextPipeline; + + get contextPipeline(): ContextPipeline { + if (this._contextPipeline === undefined) { + this._contextPipeline = this.contextPipelineBuilder(); + } + + return this._contextPipeline; + } running = false; hasFinishedBefore = false; readonly log: Log; - protected requestHandler!: RequestHandler; - protected errorHandler?: ErrorHandler; - protected failedRequestHandler?: ErrorHandler; + protected requestHandler!: RequestHandler; + protected errorHandler?: ErrorHandler; + protected failedRequestHandler?: ErrorHandler; protected requestHandlerTimeoutMillis!: number; protected internalTimeoutMillis: number; protected maxRequestRetries: number; @@ -554,7 +581,6 @@ export class BasicCrawler(); protected autoscaledPoolOptions: AutoscaledPoolOptions; protected events: EventManager; protected httpClient: BaseHttpClient; @@ -567,23 +593,22 @@ export class BasicCrawler; private _experimentWarnings: Partial> = {}; + private readonly crawlerId: string; + private readonly hasExplicitId: boolean; protected static optionsShape = { + contextPipelineBuilder: ow.optional.object, + extendContext: ow.optional.function, + requestList: ow.optional.object.validate(validators.requestList), requestQueue: ow.optional.object.validate(validators.requestQueue), // Subclasses override this function instead of passing it // in constructor, so this validation needs to apply only // if the user creates an instance of BasicCrawler directly. requestHandler: ow.optional.function, - // TODO: remove in a future release - handleRequestFunction: ow.optional.function, requestHandlerTimeoutSecs: ow.optional.number, - // TODO: remove in a future release - handleRequestTimeoutSecs: ow.optional.number, errorHandler: ow.optional.function, failedRequestHandler: ow.optional.function, - // TODO: remove in a future release - handleFailedRequestFunction: ow.optional.function, maxRequestRetries: ow.optional.number, sameDomainDelaySecs: ow.optional.number, maxSessionRotations: ow.optional.number, @@ -592,6 +617,7 @@ export class BasicCrawler = {}, + options: BasicCrawlerOptions & + RequireContextPipeline = {} as any, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX readonly config = Configuration.getGlobalConfig(), ) { ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape)); @@ -636,6 +665,7 @@ export class BasicCrawler { + let contextPipeline = (options.contextPipelineBuilder?.() ?? + ContextPipeline.create()) as ContextPipeline; // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext - handleFailedRequestFunction, - failedRequestHandler, + if (options.extendContext !== undefined) { + contextPipeline = contextPipeline.compose({ + action: async (context) => await options.extendContext(context), + }); + } - statusMessageLoggingInterval = 10, - statusMessageCallback, + contextPipeline = contextPipeline.compose({ + action: async (context) => { + const { request } = context; + if (!this.requestMatchesEnqueueStrategy(request)) { + // eslint-disable-next-line dot-notation + const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`; + this.log.debug(message); - statisticsOptions, - httpClient, - } = options; + request.noRetry = true; + request.state = RequestState.SKIPPED; + + await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); + + throw new ContextPipelineInterruptedError(message); + } + return context; + }, + }); + + return contextPipeline as ContextPipeline; + }; if (requestManager !== undefined) { if (requestList !== undefined || requestQueue !== undefined) { @@ -683,6 +744,7 @@ export class BasicCrawler (val == null ? null : +val); // allow at least 5min for internal timeouts this.internalTimeoutMillis = @@ -761,6 +789,7 @@ export class BasicCrawler maxSignedInteger) { @@ -861,15 +889,6 @@ export class BasicCrawler (this._getMessageFromError(error) as any)?.includes(x)); } - /** - * Checks whether the given crawling context is getting blocked by anti-bot protection using several heuristics. - * Returns `false` if the request is not blocked, otherwise returns a string with a description of the block reason. - * @param _crawlingContext The crawling context to check. - */ - protected async isRequestBlocked(_crawlingContext: Context): Promise { - throw new Error('the "isRequestBlocked" method is not implemented in this crawler.'); - } - /** * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds. */ @@ -1115,6 +1134,23 @@ export class BasicCrawler(defaultValue = {} as State): Promise { const kvs = await KeyValueStore.open(null, { config: this.config }); + + if (this.hasExplicitId) { + const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.crawlerId}`; + return kvs.getAutoSavedValue(stateKey, defaultValue); + } + + BasicCrawler.useStateCrawlerIds.add(this.crawlerId); + + if (BasicCrawler.useStateCrawlerIds.size > 1) { + defaultLog.warningOnce( + 'Multiple crawler instances are calling useState() without an explicit `id` option. \n' + + 'This means they will share the same state object, which is likely unintended. \n' + + 'To fix this, provide a unique `id` option to each crawler instance. \n' + + 'Example: new BasicCrawler({ id: "my-crawler-1", ... })', + ); + } + return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue); } @@ -1166,7 +1202,7 @@ export class BasicCrawler, options: CrawlerAddRequestsOptions = {}, ): Promise { await this.getRequestQueue(); @@ -1346,8 +1382,14 @@ export class BasicCrawler { - await this.requestHandler(crawlingContext as LoadedContext); + protected async runRequestHandler(crawlingContext: CrawlingContext): Promise { + await this.contextPipeline.call(crawlingContext, async (finalContext) => { + await addTimeoutToPromise( + async () => this.requestHandler(finalContext), + this.requestHandlerTimeoutMillis, + `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${finalContext.request.id}).`, + ); + }); } /** @@ -1468,12 +1510,6 @@ export class BasicCrawler { + return await this.sessionPool!.newSession({ + proxyInfo: await this.proxyConfiguration?.newProxyInfo({ + request: request ?? undefined, + }), + maxUsageCount: 1, + }); + }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`, ) @@ -1567,43 +1610,39 @@ export class BasicCrawler Promise)[] = []; + + const crawlingContext: CrawlingContext = { id: cryptoRandomObjectId(10), - crawler: this, log: this.log, request, session, - enqueueLinks: async (options: SetRequired) => { + proxyInfo: session?.proxyInfo, + enqueueLinks: async (options) => { const requestQueue = await this.getRequestQueue(); - return this.enqueueLinksWithCrawlDepth(options, request, requestQueue); + return await this.enqueueLinksWithCrawlDepth(options, request, requestQueue); }, - addRequests: async (requests: RequestsLike, options: CrawlerAddRequestsOptions = {}) => { + addRequests: async (requests, options = {}) => { const newCrawlDepth = request.crawlDepth + 1; const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth); - return this.addRequests(requestsGenerator, options); + await this.addRequests(requestsGenerator, options); }, pushData: this.pushData.bind(this), useState: this.useState.bind(this), - sendRequest: createSendRequest(this.httpClient, request, session, () => crawlingContext.proxyInfo?.url), + sendRequest: createSendRequest(this.httpClient, request!, session), getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }), + registerDeferredCleanup: (cleanup) => { + deferredCleanup.push(cleanup); + }, }; - this.crawlingContexts.set(crawlingContext.id, crawlingContext); let isRequestLocked = true; try { request.state = RequestState.REQUEST_HANDLER; - await addTimeoutToPromise( - async () => this._runRequestHandler(crawlingContext), - this.requestHandlerTimeoutMillis, - `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`, - ); + await this.runRequestHandler(crawlingContext); await this._timeoutAndRetry( async () => source.markRequestHandled(request!), @@ -1620,11 +1659,13 @@ export class BasicCrawler this._requestFunctionErrorHandler(err as Error, crawlingContext, source), + async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${ this.internalTimeoutMillis / 1e3 @@ -1635,29 +1676,29 @@ export class BasicCrawler cleanup())); // Safety net - release the lock if nobody managed to do it before if (isRequestLocked && source instanceof RequestProvider) { @@ -1694,7 +1735,7 @@ export class BasicCrawler { const { request } = crawlingContext; @@ -1790,8 +1846,9 @@ export class BasicCrawler - this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error), + await this.errorHandler?.( + crawlingContext as CrawlingContext & Partial, // valid cast - ExtendedContext transitively extends CrawlingContext + error, ); if (error instanceof SessionError) { @@ -1846,7 +1903,7 @@ export class BasicCrawler { + protected async _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise { // Always log the last error regardless if the user provided a failedRequestHandler const { id, url, method, uniqueKey } = crawlingContext.request; const message = this._getMessageFromError(error, true); @@ -1854,8 +1911,9 @@ export class BasicCrawler - this.failedRequestHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error), + await this.failedRequestHandler?.( + crawlingContext as CrawlingContext & Partial, // valid cast - ExtendedContext transitively extends CrawlingContext + error, ); } } @@ -1904,21 +1962,6 @@ export class BasicCrawler { - this.log.deprecated( - "The 'error' property of the crawling context is deprecated, and it is now passed as the second parameter in 'errorHandler' and 'failedRequestHandler'. Please update your code, as this property will be removed in a future version.", - ); - - return error; - }, - configurable: true, - }); - - return context as LoadedContext; - } - /** * Updates handledRequestsCount from possibly stored counts, usually after worker migration. */ @@ -1949,9 +1992,7 @@ export class BasicCrawler { this.events.emit(EventType.PERSIST_STATE, { isMigrating: false }); - if (this.useSessionPool) { - await this.sessionPool!.teardown(); - } + await this.sessionPool?.teardown(); if (this._closeEvents) { await this.events.close(); @@ -1960,43 +2001,6 @@ export class BasicCrawler({ - newProperty, - newName, - oldProperty, - oldName, - propertyKey, - allowUndefined = false, - }: HandlePropertyNameChangeData) { - if (newProperty && oldProperty) { - this.log.warning( - [ - `Both "${newName}" and "${oldName}" were provided in the crawler options.`, - `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`, - `As such, "${newName}" will be used instead.`, - ].join('\n'), - ); - - // @ts-expect-error Assigning to possibly readonly properties - this[propertyKey] = newProperty; - } else if (oldProperty) { - this.log.warning( - [ - `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`, - `The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`, - ].join('\n'), - ); - - // @ts-expect-error Assigning to possibly readonly properties - this[propertyKey] = oldProperty; - } else if (newProperty) { - // @ts-expect-error Assigning to possibly readonly properties - this[propertyKey] = newProperty; - } else if (!allowUndefined) { - throw new ArgumentError(`"${newName}" must be provided in the crawler options`, this.constructor); - } - } - protected _getCookieHeaderFromRequest(request: Request) { if (request.headers?.Cookie && request.headers?.cookie) { this.log.warning( @@ -2022,7 +2026,7 @@ export class BasicCrawler { - oldProperty?: Old; - newProperty?: New; - oldName: string; - newName: string; - propertyKey: string; - allowUndefined?: boolean; -} - /** * Creates new {@apilink Router} instance that works based on request labels. * This instance can then serve as a {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@apilink BasicCrawler}. diff --git a/packages/basic-crawler/src/internals/constants.ts b/packages/basic-crawler/src/internals/constants.ts deleted file mode 100644 index 56c4aa46ef95..000000000000 --- a/packages/basic-crawler/src/internals/constants.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Additional number of seconds used in {@apilink CheerioCrawler} and {@apilink BrowserCrawler} to set a reasonable - * {@apilink BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`} for {@apilink BasicCrawler} - * that would not impare functionality (not timeout before crawlers). - */ -export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = 10; diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.ts index 2e678e0e7025..e5aee681a4d0 100644 --- a/packages/basic-crawler/src/internals/send-request.ts +++ b/packages/basic-crawler/src/internals/send-request.ts @@ -1,12 +1,5 @@ -import { - type BaseHttpClient, - type HttpRequestOptions, - processHttpRequestOptions, - type Request, - type Session, -} from '@crawlee/core'; -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { GotResponse, Method } from 'got-scraping'; +import type { Request as CrawleeRequest, Session } from '@crawlee/core'; +import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@crawlee/types'; /** * Prepares a function to be used as the `sendRequest` context helper. @@ -15,40 +8,37 @@ import type { GotResponse, Method } from 'got-scraping'; * @param httpClient The HTTP client that will perform the requests. * @param originRequest The crawling request being processed. * @param session The user session associated with the current request. - * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request. */ export function createSendRequest( httpClient: BaseHttpClient, - originRequest: Request, + originRequest: CrawleeRequest, session: Session | undefined, - getProxyUrl: () => string | undefined, ) { - return async ( - // TODO the type information here (and in crawler_commons) is outright wrong... for BC - replace this with generic HttpResponse in v4 - overrideOptions: Partial = {}, - ): Promise> => { - const cookieJar = session - ? { - getCookieString: async (url: string) => session.getCookieString(url), - setCookie: async (rawCookie: string, url: string) => session.setCookie(rawCookie, url), - ...overrideOptions?.cookieJar, - } - : overrideOptions?.cookieJar; + return async ( + overrideRequest: Partial = {}, + overrideOptions: SendRequestOptions = {}, + ): Promise => { + const baseRequest = originRequest.intoFetchAPIRequest(); + const mergedUrl = overrideRequest.url ?? baseRequest.url; + const mergedMethod = overrideRequest.method ?? baseRequest.method; - const requestOptions = processHttpRequestOptions({ - url: originRequest.url, - method: originRequest.method as Method, // Narrow type to omit CONNECT - headers: originRequest.headers, - proxyUrl: getProxyUrl(), - sessionToken: session, - responseType: 'text', - ...overrideOptions, - cookieJar, - }); + const mergedHeaders = new Headers(baseRequest.headers); + if (overrideRequest.headers) { + overrideRequest.headers.forEach((value, key) => { + mergedHeaders.set(key, value); + }); + } - // Fill in body as the last step - `processHttpRequestOptions` may use either `body`, `json` or `form` so we cannot override it beforehand - requestOptions.body ??= originRequest.payload; + const request = new Request(mergedUrl, { + method: mergedMethod, + headers: mergedHeaders, + body: overrideRequest.body ?? baseRequest.body, + } as RequestInit); - return httpClient.sendRequest(requestOptions); + return httpClient.sendRequest(request, { + session, + cookieJar: overrideOptions?.cookieJar ?? session?.cookieJar, + timeout: overrideOptions.timeout, + }); }; } diff --git a/packages/basic-crawler/test/batch-add-requests.test.ts b/packages/basic-crawler/test/batch-add-requests.test.ts index 45433cdffc9c..a8628283b0e7 100644 --- a/packages/basic-crawler/test/batch-add-requests.test.ts +++ b/packages/basic-crawler/test/batch-add-requests.test.ts @@ -1,6 +1,6 @@ import { BasicCrawler } from '@crawlee/basic'; -import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js'; describe('BasicCrawler#addRequests with big batch sizes', () => { const localStorageEmulator = new MemoryStorageEmulator(); diff --git a/packages/basic-crawler/test/migration.test.ts b/packages/basic-crawler/test/migration.test.ts deleted file mode 100644 index 44cb946350ab..000000000000 --- a/packages/basic-crawler/test/migration.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -import type { Log } from '@apify/log'; -import log from '@apify/log'; - -import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator'; -import { BasicCrawler, RequestList } from '../src/index'; - -const localStorageEmulator = new MemoryStorageEmulator(); - -beforeEach(async () => { - await localStorageEmulator.init(); -}); - -afterAll(async () => { - await localStorageEmulator.destroy(); -}); - -describe('Moving from handleRequest* to requestHandler*', () => { - let requestList: RequestList; - let testLogger: Log; - - beforeEach(async () => { - requestList = await RequestList.open(null, []); - testLogger = log.child({ prefix: 'BasicCrawler' }); - }); - - describe('handleRequestFunction -> requestHandler', () => { - it('should log when providing both handleRequestFunction and requestHandler', () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: newHandler, - handleRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "requestHandler" and "handleRequestFunction" were provided in the crawler options.`, - `"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `As such, "requestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(newHandler); - }); - - it('should log when providing only the deprecated handleRequestFunction', () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - handleRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handleRequestFunction" to "requestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(oldHandler); - }); - - it('should not log when providing only requestHandler', () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(handler); - }); - }); - - describe('handleFailedRequestFunction -> failedRequestHandler', () => { - it('should log when providing both handleFailedRequestFunction and failedRequestHandler', () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - failedRequestHandler: newHandler, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`, - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `As such, "failedRequestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(newHandler); - }); - - it('should log when providing only the deprecated handleFailedRequestFunction', () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(oldHandler); - }); - - it('should not log when providing only failedRequestHandler', () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - failedRequestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(handler); - }); - }); - - describe('handleRequestTimeoutSecs -> requestHandlerTimeoutSecs', () => { - it('should log when providing both handleRequestTimeoutSecs and requestHandlerTimeoutSecs', () => { - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - requestHandlerTimeoutSecs: 420, - handleRequestTimeoutSecs: 69, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "requestHandlerTimeoutSecs" and "handleRequestTimeoutSecs" were provided in the crawler options.`, - `"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`, - `As such, "requestHandlerTimeoutSecs" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandlerTimeoutMillis']).toEqual(420_000); - }); - - it('should log when providing only the deprecated handleRequestTimeoutSecs', () => { - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - handleRequestTimeoutSecs: 69, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handleRequestTimeoutSecs" to "requestHandlerTimeoutSecs" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandlerTimeoutMillis']).toEqual(69_000); - }); - - it('should not log when providing some or no number to requestHandlerTimeoutSecs', () => { - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandlerTimeoutMillis']).toBe(60_000); - - const crawler2 = new BasicCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - requestHandlerTimeoutSecs: 420, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler2['requestHandlerTimeoutMillis']).toBe(420_000); - }); - }); -}); diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json index 9c38b8dda081..532776a32538 100644 --- a/packages/browser-crawler/package.json +++ b/packages/browser-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/browser", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,21 +40,21 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/timeout": "^0.3.0", - "@crawlee/basic": "3.15.3", - "@crawlee/browser-pool": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "ow": "^0.28.1", - "tslib": "^2.4.0", - "type-fest": "^4.0.0" + "@apify/timeout": "^0.3.2", + "@crawlee/basic": "4.0.0", + "@crawlee/browser-pool": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "ow": "^2.0.0", + "tslib": "^2.8.1", + "type-fest": "^4.41.0" }, "peerDependencies": { "playwright": "*", diff --git a/packages/browser-crawler/src/index.ts b/packages/browser-crawler/src/index.ts index d160506aadbc..0a1e8f2f4841 100644 --- a/packages/browser-crawler/src/index.ts +++ b/packages/browser-crawler/src/index.ts @@ -1,3 +1,3 @@ export * from '@crawlee/basic'; -export * from './internals/browser-crawler'; -export * from './internals/browser-launcher'; +export * from './internals/browser-crawler.js'; +export * from './internals/browser-launcher.js'; diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts index 41a8adcb5865..9af81786344e 100644 --- a/packages/browser-crawler/src/internals/browser-crawler.ts +++ b/packages/browser-crawler/src/internals/browser-crawler.ts @@ -6,19 +6,18 @@ import type { Dictionary, EnqueueLinksOptions, ErrorHandler, - LoadedContext, - ProxyConfiguration, - ProxyInfo, + LoadedRequest, + Request, RequestHandler, RequestProvider, Session, SkippedRequestCallback, } from '@crawlee/basic'; import { - BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, + ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, @@ -39,33 +38,53 @@ import type { LaunchContext, } from '@crawlee/browser-pool'; import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool'; -import type { Cookie as CookieObject } from '@crawlee/types'; +import type { BatchAddRequestsResult, Cookie as CookieObject, ProxyInfo } from '@crawlee/types'; import type { RobotsTxtFile } from '@crawlee/utils'; import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils'; import ow from 'ow'; import type { ReadonlyDeep } from 'type-fest'; -import { addTimeoutToPromise, tryCancel } from '@apify/timeout'; +import { tryCancel } from '@apify/timeout'; -import type { BrowserLaunchContext } from './browser-launcher'; +import type { BrowserLaunchContext } from './browser-launcher.js'; + +interface BaseResponse { + status(): number; +} + +type ContextDifference = Omit & Partial; export interface BrowserCrawlingContext< - Crawler = unknown, Page extends CommonPage = CommonPage, - Response = Dictionary, + Response extends BaseResponse = BaseResponse, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary, -> extends CrawlingContext { +> extends CrawlingContext { + /** + * An instance of the {@apilink BrowserController} that manages the browser instance and provides access to its API. + */ browserController: ProvidedController; + + /** + * The browser page object where the web page is loaded and rendered. + */ page: Page; - response?: Response; -} -export type BrowserRequestHandler = - RequestHandler; + /** + * The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property. + */ + request: LoadedRequest>; -export type BrowserErrorHandler = - ErrorHandler; + /** + * The HTTP response object returned by the browser's navigation. + */ + response: Response; + + /** + * Helper function for extracting URLs from the current page and adding them to the request queue. + */ + enqueueLinks: (options?: EnqueueLinksOptions) => Promise; +} export type BrowserHook = ( crawlingContext: Context, @@ -73,19 +92,25 @@ export type BrowserHook Awaitable; export interface BrowserCrawlerOptions< - Context extends BrowserCrawlingContext = BrowserCrawlingContext, + Page extends CommonPage = CommonPage, + Response extends BaseResponse = BaseResponse, + ProvidedController extends BrowserController = BrowserController, + Context extends BrowserCrawlingContext = BrowserCrawlingContext< + Page, + Response, + ProvidedController, + Dictionary + >, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>, > extends Omit< - BasicCrawlerOptions, + BasicCrawlerOptions, // Overridden with browser context - | 'requestHandler' - | 'handleRequestFunction' - | 'failedRequestHandler' - | 'handleFailedRequestFunction' - | 'errorHandler' + 'requestHandler' | 'failedRequestHandler' | 'errorHandler' > { launchContext?: BrowserLaunchContext; @@ -116,39 +141,7 @@ export interface BrowserCrawlerOptions< * The exceptions are logged to the request using the * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function. */ - requestHandler?: BrowserRequestHandler>; - - /** - * Function that is called to process each request. - * - * The function receives the {@apilink BrowserCrawlingContext} - * (actual context will be enhanced with the crawler specific properties) as an argument, where: - * - {@apilink BrowserCrawlingContext.request|`request`} is an instance of the {@apilink Request} object - * with details about the URL to open, HTTP method etc; - * - {@apilink BrowserCrawlingContext.page|`page`} is an instance of the - * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or - * Playwright [Page](https://playwright.dev/docs/api/class-page); - * - {@apilink BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@apilink BrowserController}; - * - {@apilink BrowserCrawlingContext.response|`response`} is an instance of the - * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or - * Playwright [Response](https://playwright.dev/docs/api/class-response), - * which is the main resource response as returned by the respective `page.goto()` function. - * - * The function must return a promise, which is then awaited by the crawler. - * - * If the function throws an exception, the crawler will try to re-crawl the - * request later, up to the {@apilink BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. - * If all the retries fail, the crawler calls the function - * provided to the {@apilink BrowserCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter. - * To make this work, we should **always** - * let our function throw exceptions rather than catch them. - * The exceptions are logged to the request using the - * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function. - * - * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version. - * @ignore - */ - handlePageFunction?: BrowserRequestHandler>; + requestHandler?: RequestHandler; /** * User-provided function that allows modifying the request object before it gets retried by the crawler. @@ -160,18 +153,7 @@ export interface BrowserCrawlerOptions< * Second argument is the `Error` instance that * represents the last error thrown during processing of the request. */ - errorHandler?: BrowserErrorHandler; - - /** - * A function to handle requests that failed more than `option.maxRequestRetries` times. - * - * The function receives the {@apilink BrowserCrawlingContext} - * (actual context will be enhanced with the crawler specific properties) as the first argument, - * where the {@apilink BrowserCrawlingContext.request|`request`} corresponds to the failed request. - * Second argument is the `Error` instance that - * represents the last error thrown during processing of the request. - */ - failedRequestHandler?: BrowserErrorHandler; + errorHandler?: ErrorHandler; /** * A function to handle requests that failed more than `option.maxRequestRetries` times. @@ -181,11 +163,8 @@ export interface BrowserCrawlerOptions< * where the {@apilink BrowserCrawlingContext.request|`request`} corresponds to the failed request. * Second argument is the `Error` instance that * represents the last error thrown during processing of the request. - * - * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version. - * @ignore */ - handleFailedRequestFunction?: BrowserErrorHandler; + failedRequestHandler?: ErrorHandler; /** * Custom options passed to the underlying {@apilink BrowserPool} constructor. @@ -194,12 +173,6 @@ export interface BrowserCrawlerOptions< browserPoolOptions?: Partial & Partial>; - /** - * If set, the crawler will be configured for all connections to use - * the Proxy URLs provided and rotated according to the configuration. - */ - proxyConfiguration?: ProxyConfiguration; - /** * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, @@ -312,17 +285,21 @@ export interface BrowserCrawlerOptions< * @category Crawlers */ export abstract class BrowserCrawler< + Page extends CommonPage = CommonPage, + Response extends BaseResponse = BaseResponse, + ProvidedController extends BrowserController = BrowserController, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, - Context extends BrowserCrawlingContext = BrowserCrawlingContext, + Context extends BrowserCrawlingContext = BrowserCrawlingContext< + Page, + Response, + ProvidedController, + Dictionary + >, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, GoToOptions extends Dictionary = Dictionary, -> extends BasicCrawler { - /** - * A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies. - * Only available if used by the crawler. - */ - proxyConfiguration?: ProxyConfiguration; - +> extends BasicCrawler { /** * A reference to the underlying {@apilink BrowserPool} class that manages the crawler's browsers. */ @@ -330,16 +307,16 @@ export abstract class BrowserCrawler< launchContext: BrowserLaunchContext; - protected userProvidedRequestHandler!: BrowserRequestHandler; + protected readonly ignoreShadowRoots: boolean; + protected readonly ignoreIframes: boolean; + protected navigationTimeoutMillis: number; - protected requestHandlerTimeoutInnerMillis: number; protected preNavigationHooks: BrowserHook[]; protected postNavigationHooks: BrowserHook[]; protected persistCookiesPerSession: boolean; protected static override optionsShape = { ...BasicCrawler.optionsShape, - handlePageFunction: ow.optional.function, navigationTimeoutSecs: ow.optional.number.greaterThan(0), preNavigationHooks: ow.optional.array, @@ -352,73 +329,54 @@ export abstract class BrowserCrawler< persistCookiesPerSession: ow.optional.boolean, useSessionPool: ow.optional.boolean, proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration), - ignoreShadowRoots: ow.optional.boolean, - ignoreIframes: ow.optional.boolean, }; /** * All `BrowserCrawler` parameters are passed via an options object. */ protected constructor( - options: BrowserCrawlerOptions = {}, + options: BrowserCrawlerOptions< + Page, + Response, + ProvidedController, + Context, + ContextExtension, + ExtendedContext + > & { + contextPipelineBuilder: () => ContextPipeline; + }, override readonly config = Configuration.getGlobalConfig(), ) { ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape)); const { navigationTimeoutSecs = 60, - requestHandlerTimeoutSecs = 60, persistCookiesPerSession, - proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], - // Ignored - handleRequestFunction, - - requestHandler: userProvidedRequestHandler, - handlePageFunction, - - failedRequestHandler, - handleFailedRequestFunction, headless, - ignoreShadowRoots, - ignoreIframes, + ignoreIframes = false, + ignoreShadowRoots = false, + contextPipelineBuilder, + extendContext, + proxyConfiguration, ...basicCrawlerOptions } = options; super( { ...basicCrawlerOptions, - requestHandler: async (...args) => this._runRequestHandler(...(args as [Context])), - requestHandlerTimeoutSecs: - navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, + contextPipelineBuilder: () => + contextPipelineBuilder() + .compose({ action: this.performNavigation.bind(this) }) + .compose({ action: this.handleBlockedRequestByContent.bind(this) }) + .compose({ action: this.restoreRequestState.bind(this) }), + extendContext: extendContext as (context: Context) => Awaitable, }, config, ); - this._handlePropertyNameChange({ - newName: 'requestHandler', - oldName: 'handlePageFunction', - propertyKey: 'userProvidedRequestHandler', - newProperty: userProvidedRequestHandler, - oldProperty: handlePageFunction, - allowUndefined: true, // fallback to the default router - }); - - if (!this.userProvidedRequestHandler) { - this.userProvidedRequestHandler = this.router; - } - - this._handlePropertyNameChange({ - newName: 'failedRequestHandler', - oldName: 'handleFailedRequestFunction', - propertyKey: 'failedRequestHandler', - newProperty: failedRequestHandler, - oldProperty: handleFailedRequestFunction, - allowUndefined: true, - }); - // Cookies should be persisted per session only if session pool is used if (!this.useSessionPool && persistCookiesPerSession) { throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.'); @@ -426,10 +384,11 @@ export abstract class BrowserCrawler< this.launchContext = launchContext; this.navigationTimeoutMillis = navigationTimeoutSecs * 1000; - this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000; this.proxyConfiguration = proxyConfiguration; this.preNavigationHooks = preNavigationHooks; this.postNavigationHooks = postNavigationHooks; + this.ignoreIframes = ignoreIframes; + this.ignoreShadowRoots = ignoreShadowRoots; if (headless != null) { this.launchContext.launchOptions ??= {} as LaunchOptions; @@ -457,13 +416,23 @@ export abstract class BrowserCrawler< }); } - protected override async _cleanupContext(crawlingContext: Context): Promise { - const { page } = crawlingContext; - - // Page creation may be aborted - if (page) { - await page.close().catch((error: Error) => this.log.debug('Error while closing page', { error })); - } + protected buildContextPipeline(): ContextPipeline< + CrawlingContext, + BrowserCrawlingContext + > { + return ContextPipeline.create().compose({ + action: this.preparePage.bind(this), + cleanup: async (context: { + page: Page; + registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup']; + }) => { + context.registerDeferredCleanup(async () => { + await context.page + .close() + .catch((error: Error) => this.log.debug('Error while closing page', { error })); + }); + }, + }); } private async containsSelectors(page: CommonPage, selectors: string[]): Promise { @@ -475,7 +444,9 @@ export abstract class BrowserCrawler< return foundSelectors.length > 0 ? foundSelectors : null; } - protected override async isRequestBlocked(crawlingContext: Context): Promise { + protected async isRequestBlocked( + crawlingContext: BrowserCrawlingContext, + ): Promise { const { page, response } = crawlingContext; const blockedStatusCodes = @@ -505,29 +476,25 @@ export abstract class BrowserCrawler< return false; } - /** - * Wrapper around requestHandler that opens and closes pages etc. - */ - protected override async _runRequestHandler(crawlingContext: Context) { + private async preparePage( + crawlingContext: CrawlingContext, + ): Promise< + ContextDifference> + > { const newPageOptions: Dictionary = { id: crawlingContext.id, }; const useIncognitoPages = this.launchContext?.useIncognitoPages; - const experimentalContainers = this.launchContext?.experimentalContainers; - if (this.proxyConfiguration) { - const { session } = crawlingContext; - - const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, { - request: crawlingContext.request, - }); + if (crawlingContext.session?.proxyInfo) { + const proxyInfo = crawlingContext.session.proxyInfo; crawlingContext.proxyInfo = proxyInfo; newPageOptions.proxyUrl = proxyInfo?.url; newPageOptions.proxyTier = proxyInfo?.proxyTier; - if (this.proxyConfiguration.isManInTheMiddle) { + if (proxyInfo?.ignoreTlsErrors) { /** * @see https://playwright.dev/docs/api/class-browser/#browser-new-context * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md @@ -539,107 +506,66 @@ export abstract class BrowserCrawler< } } - const page = (await this.browserPool.newPage(newPageOptions)) as CommonPage; - tryCancel(); - this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages || experimentalContainers); - - // DO NOT MOVE THIS LINE ABOVE! - // `enhanceCrawlingContextWithPageInfo` gives us a valid session. - // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`. - // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice. - const { request, session } = crawlingContext; - - if (!request.skipNavigation) { - await this._handleNavigation(crawlingContext); - tryCancel(); - - await this._responseHandler(crawlingContext); - tryCancel(); - - // save cookies - // TODO: Should we save the cookies also after/only the handle page? - if (this.persistCookiesPerSession) { - const cookies = await crawlingContext.browserController.getCookies(page); - tryCancel(); - session?.setCookies(cookies, request.loadedUrl!); - } - } - - if (!this.requestMatchesEnqueueStrategy(request)) { - this.log.debug( - // eslint-disable-next-line dot-notation - `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`, - ); - - request.noRetry = true; - request.state = RequestState.SKIPPED; - - await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); - - return; - } - - if (this.retryOnBlocked) { - const error = await this.isRequestBlocked(crawlingContext); - if (error) throw new SessionError(error); - } - - request.state = RequestState.REQUEST_HANDLER; - try { - await addTimeoutToPromise( - async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext as LoadedContext)), - this.requestHandlerTimeoutInnerMillis, - `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`, - ); - - request.state = RequestState.DONE; - } catch (e: any) { - request.state = RequestState.ERROR; - throw e; - } + const page = (await this.browserPool.newPage(newPageOptions)) as Page; tryCancel(); - } - protected _enhanceCrawlingContextWithPageInfo( - crawlingContext: Context, - page: CommonPage, - createNewSession?: boolean, - ): void { - crawlingContext.page = page; - - // This switch is because the crawlingContexts are created on per request basis. - // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration. - // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler, - // which configures proxy and session for every new request const browserControllerInstance = this.browserPool.getBrowserControllerByPage( page as any, - ) as Context['browserController']; - crawlingContext.browserController = browserControllerInstance; + ) as ProvidedController; - if (!createNewSession) { - crawlingContext.session = browserControllerInstance.launchContext.session as Session; - } + const contextEnqueueLinks = crawlingContext.enqueueLinks; - if (!crawlingContext.proxyInfo) { - crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo as ProxyInfo; - } + const session = useIncognitoPages + ? crawlingContext.session + : (browserControllerInstance.launchContext.session as Session); - const contextEnqueueLinks = crawlingContext.enqueueLinks; - crawlingContext.enqueueLinks = async (enqueueOptions) => { - return browserCrawlerEnqueueLinks({ - options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) }, - page, - requestQueue: await this.getRequestQueue(), - robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url), - onSkippedRequest: this.handleSkippedRequest, - originalRequestUrl: crawlingContext.request.url, - finalRequestUrl: crawlingContext.request.loadedUrl, - enqueueLinks: contextEnqueueLinks, - }); + return { + page, + get response(): Response { + throw new Error( + "The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)", + ); + }, + browserController: browserControllerInstance, + session, + proxyInfo: session?.proxyInfo, + enqueueLinks: async (enqueueOptions: EnqueueLinksOptions = {}) => { + return (await browserCrawlerEnqueueLinks({ + options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) }, + page, + requestQueue: await this.getRequestQueue(), + robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url), + onSkippedRequest: this.handleSkippedRequest, + originalRequestUrl: crawlingContext.request.url, + finalRequestUrl: crawlingContext.request.loadedUrl, + enqueueLinks: contextEnqueueLinks, + })) as BatchAddRequestsResult; // TODO make this type safe + }, }; } - protected async _handleNavigation(crawlingContext: Context) { + private async performNavigation(crawlingContext: Context): Promise<{ + request: LoadedRequest; + response?: Response; + }> { + if (crawlingContext.request.skipNavigation) { + return { + request: new Proxy(crawlingContext.request, { + get(target, propertyName, receiver) { + if (propertyName === 'loadedUrl') { + throw new Error( + 'The `request.loadedUrl` property is not available - `skipNavigation` was used', + ); + } + return Reflect.get(target, propertyName, receiver); + }, + }) as LoadedRequest, + get response(): Response { + throw new Error('The `response` property is not available - `skipNavigation` was used'); + }, + }; + } + const gotoOptions = { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions; const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request); @@ -652,8 +578,10 @@ export abstract class BrowserCrawler< await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies); + let response: Response | undefined; + try { - crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined; + response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined; } catch (error) { await this._handleNavigationTimeout(crawlingContext, error as Error); @@ -666,10 +594,48 @@ export abstract class BrowserCrawler< crawlingContext.request.state = RequestState.AFTER_NAV; await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions); + + await this.processResponse(response, crawlingContext); + tryCancel(); + + // save cookies + // TODO: Should we save the cookies also after/only the handle page? + if (this.persistCookiesPerSession) { + const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page); + tryCancel(); + crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl!); + } + + if (response !== undefined) { + return { + request: crawlingContext.request as LoadedRequest, + response, + }; + } + + return { + request: crawlingContext.request as LoadedRequest, + }; + } + + private async handleBlockedRequestByContent( + crawlingContext: BrowserCrawlingContext, + ) { + if (this.retryOnBlocked) { + const error = await this.isRequestBlocked(crawlingContext); + if (error) throw new SessionError(error); + } + + return {}; + } + + private async restoreRequestState(crawlingContext: CrawlingContext) { + crawlingContext.request.state = RequestState.REQUEST_HANDLER; + return {}; } protected async _applyCookies( - { session, request, page, browserController }: Context, + { session, request, page, browserController }: BrowserCrawlingContext, preHooksCookies: string, postHooksCookies: string, ) { @@ -688,7 +654,7 @@ export abstract class BrowserCrawler< /** * Marks session bad in case of navigation timeout. */ - protected async _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise { + protected async _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise { const { session } = crawlingContext; if (error && error.constructor.name === 'TimeoutError') { @@ -708,15 +674,15 @@ export abstract class BrowserCrawler< } protected abstract _navigationHandler( - crawlingContext: Context, + crawlingContext: BrowserCrawlingContext, gotoOptions: GoToOptions, ): Promise; - /** - * Should be overridden in case of different automation library that does not support this response API. - */ - protected async _responseHandler(crawlingContext: Context): Promise { - const { response, session, request, page } = crawlingContext; + private async processResponse( + response: Response | undefined, + crawlingContext: BrowserCrawlingContext, + ): Promise { + const { session, request, page } = crawlingContext; if (typeof response === 'object' && typeof response.status === 'function') { const status: number = response.status(); @@ -739,18 +705,21 @@ export abstract class BrowserCrawler< const launchContextExtends: { session?: Session; proxyInfo?: ProxyInfo } = {}; if (this.sessionPool) { - launchContextExtends.session = await this.sessionPool.getSession(); + launchContextExtends.session = await this.sessionPool.newSession({ + proxyInfo: await this.proxyConfiguration?.newProxyInfo({ + // cannot pass a request here, since session is created on browser launch + }), + }); } - if (this.proxyConfiguration && !launchContext.proxyUrl) { - const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, { - proxyTier: (launchContext.proxyTier as number) ?? undefined, - }); + if (!launchContext.proxyUrl && launchContextExtends.session?.proxyInfo) { + const proxyInfo = launchContextExtends.session.proxyInfo; + launchContext.proxyUrl = proxyInfo?.url; launchContextExtends.proxyInfo = proxyInfo; // Disable SSL verification for MITM proxies - if (this.proxyConfiguration.isManInTheMiddle) { + if (proxyInfo?.ignoreTlsErrors) { /** * @see https://playwright.dev/docs/api/class-browser/#browser-new-context * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md @@ -846,6 +815,7 @@ export async function browserCrawlerEnqueueLinks( ...enqueueLinksOptions, }); } + return enqueueLinks({ requestQueue: options.requestQueue, robotsTxtFile: options.robotsTxtFile, diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.ts index 1b68b1bb3353..3c799677ac1c 100644 --- a/packages/browser-crawler/src/internals/browser-launcher.ts +++ b/packages/browser-crawler/src/internals/browser-launcher.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { createRequire } from 'node:module'; import os from 'node:os'; import { Configuration } from '@crawlee/basic'; @@ -11,6 +12,8 @@ const DEFAULT_VIEWPORT = { height: 768, }; +const require = createRequire(import.meta.url); + export interface BrowserLaunchContext extends BrowserPluginOptions { /** * URL to an HTTP proxy server. It must define the port number, @@ -46,13 +49,6 @@ export interface BrowserLaunchContext extends BrowserPluginO */ useIncognitoPages?: boolean; - /** - * @experimental - * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading. - * Works best with Firefox. Unstable on Chromium. - */ - experimentalContainers?: boolean; - /** * Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path. * The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state. @@ -107,7 +103,6 @@ export abstract class BrowserLauncher< useChrome: ow.optional.boolean, useIncognitoPages: ow.optional.boolean, browserPerProxy: ow.optional.boolean, - experimentalContainers: ow.optional.boolean, userDataDir: ow.optional.string, launchOptions: ow.optional.object, userAgent: ow.optional.string, diff --git a/packages/browser-crawler/test/migration.test.ts b/packages/browser-crawler/test/migration.test.ts deleted file mode 100644 index af683550bd15..000000000000 --- a/packages/browser-crawler/test/migration.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { PuppeteerPlugin } from '@crawlee/browser-pool'; -import puppeteer from 'puppeteer'; - -import type { Log } from '@apify/log'; -import log from '@apify/log'; - -import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator'; -import { BrowserCrawler, RequestList } from '../src/index'; - -const localStorageEmulator = new MemoryStorageEmulator(); - -beforeEach(async () => { - await localStorageEmulator.init(); -}); - -afterAll(async () => { - await localStorageEmulator.destroy(); -}); - -const plugin = new PuppeteerPlugin(puppeteer); - -describe('Moving from handleRequest* to requestHandler*', () => { - let requestList: RequestList; - let testLogger: Log; - - beforeEach(async () => { - requestList = await RequestList.open(null, []); - testLogger = log.child({ prefix: 'BrowserCrawler' }); - }); - - describe('handlePageFunction -> requestHandler', () => { - it('should log when providing both handlePageFunction and requestHandler', async () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- Protected constructor - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - requestHandler: newHandler, - handlePageFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`, - `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `As such, "requestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['userProvidedRequestHandler']).toBe(newHandler); - - await crawler.browserPool.destroy(); - }); - - it('should log when providing only the deprecated handlePageFunction', async () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- We are verifying the deprecation warning - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - handlePageFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['userProvidedRequestHandler']).toBe(oldHandler); - - await crawler.browserPool.destroy(); - }); - - it('should not log when providing only requestHandler', async () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- Protected constructor - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - requestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['userProvidedRequestHandler']).toBe(handler); - - await crawler.browserPool.destroy(); - }); - }); - - describe('handleFailedRequestFunction -> failedRequestHandler', () => { - it('should log when providing both handleFailedRequestFunction and failedRequestHandler', async () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- Protected constructor - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - requestHandler: () => {}, - failedRequestHandler: newHandler, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`, - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `As such, "failedRequestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(newHandler); - - await crawler.browserPool.destroy(); - }); - - it('should log when providing only the deprecated handleFailedRequestFunction', async () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- Protected constructor - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - requestHandler: () => {}, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(oldHandler); - - await crawler.browserPool.destroy(); - }); - - it('should not log when providing only failedRequestHandler', async () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - // @ts-expect-error -- Protected constructor - const crawler = new BrowserCrawler({ - requestList, - log: testLogger, - browserPoolOptions: { - browserPlugins: [plugin], - }, - requestHandler: () => {}, - failedRequestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(handler); - - await crawler.browserPool.destroy(); - }); - }); -}); diff --git a/packages/browser-crawler/test/tsconfig.json b/packages/browser-crawler/test/tsconfig.json deleted file mode 100644 index bf55f9516b7d..000000000000 --- a/packages/browser-crawler/test/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "include": ["**/*", "../../**/*"], - "compilerOptions": { - "types": ["vitest/globals"] - } -} diff --git a/packages/browser-pool/copy-definitions.mjs b/packages/browser-pool/copy-definitions.mjs deleted file mode 100644 index 797e62a13e10..000000000000 --- a/packages/browser-pool/copy-definitions.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { copyFileSync, mkdirSync, readdirSync } from 'node:fs'; -import { join } from 'node:path'; - -const copyFolderSync = (from, to) => { - mkdirSync(to); - - for (const file of readdirSync(from, { withFileTypes: true })) { - if (file.isDirectory()) { - copyFolderSync(join(from, file.name), join(to, file.name)); - } else if (file.isFile()) { - copyFileSync(join(from, file.name), join(to, file.name)); - } - } -}; - -copyFolderSync('tab-as-a-container', 'dist/tab-as-a-container'); diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json index 164fac854c2c..c92ae0e14c80 100644 --- a/packages/browser-pool/package.json +++ b/packages/browser-pool/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/browser-pool", - "version": "3.15.3", + "version": "4.0.0", "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "author": { @@ -30,26 +24,26 @@ "url": "https://github.com/apify/crawlee/issues" }, "scripts": { - "build": "yarn clean && yarn compile && node copy-definitions.mjs && yarn copy", + "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "dependencies": { - "@apify/log": "^2.4.0", - "@apify/timeout": "^0.3.0", - "@crawlee/core": "3.15.3", - "@crawlee/types": "3.15.3", + "@apify/log": "^2.5.18", + "@apify/timeout": "^0.3.2", + "@crawlee/core": "4.0.0", + "@crawlee/types": "4.0.0", "fingerprint-generator": "^2.1.68", "fingerprint-injector": "^2.1.68", "lodash.merge": "^4.6.2", - "nanoid": "^3.3.4", - "ow": "^0.28.1", - "p-limit": "^3.1.0", - "proxy-chain": "^2.0.1", - "quick-lru": "^5.1.1", + "nanoid": "^5.1.5", + "ow": "^2.0.0", + "p-limit": "^6.2.0", + "proxy-chain": "^2.5.8", + "quick-lru": "^7.0.1", "tiny-typed-emitter": "^2.1.0", - "tslib": "^2.4.0" + "tslib": "^2.8.1" }, "peerDependencies": { "playwright": "*", diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.ts index 7a4e796f3880..0c546488ed7f 100644 --- a/packages/browser-pool/src/abstract-classes/browser-controller.ts +++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts @@ -4,11 +4,11 @@ import { TypedEmitter } from 'tiny-typed-emitter'; import { tryCancel } from '@apify/timeout'; -import { BROWSER_CONTROLLER_EVENTS } from '../events'; -import type { LaunchContext } from '../launch-context'; -import { log } from '../logger'; -import type { UnwrapPromise } from '../utils'; -import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin'; +import { BROWSER_CONTROLLER_EVENTS } from '../events.js'; +import type { LaunchContext } from '../launch-context.js'; +import { log } from '../logger.js'; +import type { UnwrapPromise } from '../utils.js'; +import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js'; const PROCESS_KILL_TIMEOUT_MILLIS = 5000; diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts index 7b95dd7554cf..3eb69e011a99 100644 --- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts +++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts @@ -2,10 +2,10 @@ import { CriticalError } from '@crawlee/core'; import type { Dictionary } from '@crawlee/types'; import merge from 'lodash.merge'; -import type { LaunchContextOptions } from '../launch-context'; -import { LaunchContext } from '../launch-context'; -import type { UnwrapPromise } from '../utils'; -import type { BrowserController } from './browser-controller'; +import type { LaunchContextOptions } from '../launch-context.js'; +import { LaunchContext } from '../launch-context.js'; +import type { UnwrapPromise } from '../utils.js'; +import type { BrowserController } from './browser-controller.js'; /** * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer' @@ -65,12 +65,6 @@ export interface BrowserPluginOptions { * @default false */ useIncognitoPages?: boolean; - /** - * @experimental - * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading. - * Works best with Firefox. Unstable on Chromium. - */ - experimentalContainers?: boolean; /** * Path to a User Data Directory, which stores browser session data like cookies and local storage. */ @@ -111,19 +105,11 @@ export abstract class BrowserPlugin< NewPageResult = UnwrapPromise>, > { name = this.constructor.name; - library: Library; - launchOptions: LibraryOptions; - proxyUrl?: string; - userDataDir?: string; - useIncognitoPages: boolean; - - experimentalContainers: boolean; - browserPerProxy?: boolean; constructor(library: Library, options: BrowserPluginOptions = {}) { @@ -132,7 +118,6 @@ export abstract class BrowserPlugin< proxyUrl, userDataDir, useIncognitoPages = false, - experimentalContainers = false, browserPerProxy = false, } = options; @@ -141,7 +126,6 @@ export abstract class BrowserPlugin< this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1); this.userDataDir = userDataDir; this.useIncognitoPages = useIncognitoPages; - this.experimentalContainers = experimentalContainers; this.browserPerProxy = browserPerProxy; } @@ -160,7 +144,6 @@ export abstract class BrowserPlugin< proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, - experimentalContainers = this.experimentalContainers, browserPerProxy = this.browserPerProxy, proxyTier, } = options; @@ -171,16 +154,19 @@ export abstract class BrowserPlugin< browserPlugin: this, proxyUrl, useIncognitoPages, - experimentalContainers, userDataDir, browserPerProxy, proxyTier, }); } - createController(): BrowserController { - return this._createController(); - } + abstract createController(): BrowserController< + Library, + LibraryOptions, + LaunchResult, + NewPageOptions, + NewPageResult + >; /** * Launches the browser using provided launch context. @@ -278,17 +264,6 @@ export abstract class BrowserPlugin< protected abstract _launch( launchContext: LaunchContext, ): Promise; - - /** - * @private - */ - protected abstract _createController(): BrowserController< - Library, - LibraryOptions, - LaunchResult, - NewPageOptions, - NewPageResult - >; } export class BrowserLaunchError extends CriticalError { diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts index 8c26ee8b804e..6da577a48037 100644 --- a/packages/browser-pool/src/browser-pool.ts +++ b/packages/browser-pool/src/browser-pool.ts @@ -10,18 +10,18 @@ import { TypedEmitter } from 'tiny-typed-emitter'; import { addTimeoutToPromise, tryCancel } from '@apify/timeout'; -import type { BrowserController } from './abstract-classes/browser-controller'; -import type { BrowserPlugin } from './abstract-classes/browser-plugin'; -import { BROWSER_POOL_EVENTS } from './events'; +import type { BrowserController } from './abstract-classes/browser-controller.js'; +import type { BrowserPlugin } from './abstract-classes/browser-plugin.js'; +import { BROWSER_POOL_EVENTS } from './events.js'; import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, -} from './fingerprinting/hooks'; -import type { FingerprintGeneratorOptions } from './fingerprinting/types'; -import type { LaunchContext } from './launch-context'; -import { log } from './logger'; -import type { InferBrowserPluginArray, UnwrapPromise } from './utils'; +} from './fingerprinting/hooks.js'; +import type { FingerprintGeneratorOptions } from './fingerprinting/types.js'; +import type { LaunchContext } from './launch-context.js'; +import { log } from './logger.js'; +import type { InferBrowserPluginArray, UnwrapPromise } from './utils.js'; const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000; const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000; @@ -558,10 +558,7 @@ export class BrowserPool< await browserController['isActivePromise']; tryCancel(); - const finalPageOptions = - browserController.launchContext.useIncognitoPages || browserController.launchContext.experimentalContainers - ? pageOptions - : undefined; + const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined; if (finalPageOptions) { Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions)); diff --git a/packages/browser-pool/src/fingerprinting/hooks.ts b/packages/browser-pool/src/fingerprinting/hooks.ts index 3a8e83724d2f..1e22b72de411 100644 --- a/packages/browser-pool/src/fingerprinting/hooks.ts +++ b/packages/browser-pool/src/fingerprinting/hooks.ts @@ -1,12 +1,12 @@ import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator'; import type { FingerprintInjector } from 'fingerprint-injector'; -import type { BrowserController } from '../abstract-classes/browser-controller'; -import type { BrowserPool } from '../browser-pool'; -import type { LaunchContext } from '../launch-context'; -import { PlaywrightPlugin } from '../playwright/playwright-plugin'; -import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin'; -import { getGeneratorDefaultOptions } from './utils'; +import type { BrowserController } from '../abstract-classes/browser-controller.js'; +import type { BrowserPool } from '../browser-pool.js'; +import type { LaunchContext } from '../launch-context.js'; +import { PlaywrightPlugin } from '../playwright/playwright-plugin.js'; +import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js'; +import { getGeneratorDefaultOptions } from './utils.js'; /** * @internal diff --git a/packages/browser-pool/src/fingerprinting/utils.ts b/packages/browser-pool/src/fingerprinting/utils.ts index 5efd4b7deb2a..07f45acef819 100644 --- a/packages/browser-pool/src/fingerprinting/utils.ts +++ b/packages/browser-pool/src/fingerprinting/utils.ts @@ -1,9 +1,9 @@ -import type { BrowserPlugin } from '../abstract-classes/browser-plugin'; -import type { LaunchContext } from '../launch-context'; -import { PlaywrightPlugin } from '../playwright/playwright-plugin'; -import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin'; -import type { FingerprintGeneratorOptions } from './types'; -import { BrowserName, DeviceCategory, OperatingSystemsName } from './types'; +import type { BrowserPlugin } from '../abstract-classes/browser-plugin.js'; +import type { LaunchContext } from '../launch-context.js'; +import { PlaywrightPlugin } from '../playwright/playwright-plugin.js'; +import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js'; +import type { FingerprintGeneratorOptions } from './types.js'; +import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js'; export const getGeneratorDefaultOptions = (launchContext: LaunchContext): FingerprintGeneratorOptions => { const { browserPlugin, launchOptions } = launchContext; diff --git a/packages/browser-pool/src/index.ts b/packages/browser-pool/src/index.ts index d3b4d24619d8..1e7295bd943d 100644 --- a/packages/browser-pool/src/index.ts +++ b/packages/browser-pool/src/index.ts @@ -22,19 +22,19 @@ * * @module browser-pool */ -export * from './browser-pool'; -export * from './playwright/playwright-plugin'; -export * from './puppeteer/puppeteer-plugin'; -export * from './events'; +export * from './browser-pool.js'; +export * from './playwright/playwright-plugin.js'; +export * from './puppeteer/puppeteer-plugin.js'; +export * from './events.js'; export { BrowserName, DeviceCategory, OperatingSystemsName, -} from './fingerprinting/types'; -export { BrowserController, BrowserControllerEvents } from './abstract-classes/browser-controller'; -export { PuppeteerController } from './puppeteer/puppeteer-controller'; -export { PlaywrightController } from './playwright/playwright-controller'; -export { PlaywrightBrowser } from './playwright/playwright-browser'; +} from './fingerprinting/types.js'; +export { BrowserController, BrowserControllerEvents } from './abstract-classes/browser-controller.js'; +export { PuppeteerController } from './puppeteer/puppeteer-controller.js'; +export { PlaywrightController } from './playwright/playwright-controller.js'; +export { PlaywrightBrowser } from './playwright/playwright-browser.js'; export { CommonPage, CommonLibrary, @@ -43,12 +43,12 @@ export { CreateLaunchContextOptions, BrowserLaunchError, DEFAULT_USER_AGENT, -} from './abstract-classes/browser-plugin'; -export { LaunchContext, LaunchContextOptions } from './launch-context'; +} from './abstract-classes/browser-plugin.js'; +export { LaunchContext, LaunchContextOptions } from './launch-context.js'; export { BrowserSpecification, FingerprintGenerator, FingerprintGeneratorOptions, GetFingerprintReturn, -} from './fingerprinting/types'; -export { InferBrowserPluginArray, UnwrapPromise } from './utils'; +} from './fingerprinting/types.js'; +export { InferBrowserPluginArray, UnwrapPromise } from './utils.js'; diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.ts index e7cbdfbb4aab..2820d63ee13d 100644 --- a/packages/browser-pool/src/launch-context.ts +++ b/packages/browser-pool/src/launch-context.ts @@ -1,8 +1,8 @@ import type { Dictionary } from '@crawlee/types'; import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator'; -import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './abstract-classes/browser-plugin'; -import type { UnwrapPromise } from './utils'; +import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './abstract-classes/browser-plugin.js'; +import type { UnwrapPromise } from './utils.js'; /** * `LaunchContext` holds information about the launched browser. It's useful @@ -46,12 +46,6 @@ export interface LaunchContextOptions< * If set to `true` each page uses its own context that is destroyed once the page is closed or crashes. */ useIncognitoPages?: boolean; - /** - * @experimental - * Like `useIncognitoPages`, but for persistent contexts, so cache is used for faster loading. - * Works best with Firefox. Unstable on Chromium. - */ - experimentalContainers?: boolean; /** * Path to a User Data Directory, which stores browser session data like cookies and local storage. */ @@ -72,7 +66,6 @@ export class LaunchContext< launchOptions: LibraryOptions; useIncognitoPages: boolean; browserPerProxy?: boolean; - experimentalContainers: boolean; userDataDir: string; proxyTier?: number; @@ -90,7 +83,6 @@ export class LaunchContext< proxyUrl, useIncognitoPages, browserPerProxy, - experimentalContainers, userDataDir = '', proxyTier, } = options; @@ -100,7 +92,6 @@ export class LaunchContext< this.launchOptions = launchOptions; this.browserPerProxy = browserPerProxy ?? false; this.useIncognitoPages = useIncognitoPages ?? false; - this.experimentalContainers = experimentalContainers ?? false; this.userDataDir = userDataDir; this.proxyTier = proxyTier; diff --git a/packages/browser-pool/src/playwright/load-firefox-addon.ts b/packages/browser-pool/src/playwright/load-firefox-addon.ts deleted file mode 100644 index a11960248ec8..000000000000 --- a/packages/browser-pool/src/playwright/load-firefox-addon.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { Buffer } from 'node:buffer'; -import net from 'node:net'; - -export const loadFirefoxAddon = async (port: number, host: string, addonPath: string) => { - return new Promise((resolve) => { - const socket = net.connect({ - port, - host, - }); - - let success = false; - - socket.once('error', () => {}); - socket.once('close', () => { - resolve(success); - }); - - const send = (data: Record) => { - const raw = Buffer.from(JSON.stringify(data)); - - socket.write(`${raw.length}`); - socket.write(':'); - socket.write(raw); - }; - - send({ - to: 'root', - type: 'getRoot', - }); - - const onMessage = (message: any) => { - if (message.addonsActor) { - send({ - to: message.addonsActor, - type: 'installTemporaryAddon', - addonPath, - }); - } - - if (message.addon) { - success = true; - socket.end(); - } - - if (message.error) { - socket.end(); - } - }; - - const buffers: Buffer[] = []; - let remainingBytes = 0; - - socket.on('data', (data) => { - while (true) { - if (remainingBytes === 0) { - const index = data.indexOf(':'); - - buffers.push(data); - - if (index === -1) { - return; - } - - const buffer = Buffer.concat(buffers); - const bufferIndex = buffer.indexOf(':'); - - buffers.length = 0; - remainingBytes = Number(buffer.subarray(0, bufferIndex).toString()); - - if (!Number.isFinite(remainingBytes)) { - throw new Error('Invalid state'); - } - - data = buffer.subarray(bufferIndex + 1); - } - - if (data.length < remainingBytes) { - remainingBytes -= data.length; - buffers.push(data); - break; - } - - buffers.push(data.subarray(0, remainingBytes)); - - const buffer = Buffer.concat(buffers); - buffers.length = 0; - - const json = JSON.parse(buffer.toString()); - queueMicrotask(() => { - onMessage(json); - }); - - const remainder = data.subarray(remainingBytes); - remainingBytes = 0; - - if (remainder.length === 0) { - break; - } - - data = remainder; - } - }); - }); -}; diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.ts index 7dc27f7c4102..6761f3055e01 100644 --- a/packages/browser-pool/src/playwright/playwright-controller.ts +++ b/packages/browser-pool/src/playwright/playwright-controller.ts @@ -3,13 +3,9 @@ import type { Browser, BrowserType, Page } from 'playwright'; import { tryCancel } from '@apify/timeout'; -import { BrowserController } from '../abstract-classes/browser-controller'; -import { anonymizeProxySugar } from '../anonymize-proxy'; -import type { SafeParameters } from '../utils'; -import type { PlaywrightPlugin } from './playwright-plugin'; - -const tabIds = new WeakMap(); -const keyFromTabId = (tabId: string | number) => `.${tabId}.`; +import { BrowserController } from '../abstract-classes/browser-controller.js'; +import { anonymizeProxySugar } from '../anonymize-proxy.js'; +import type { SafeParameters } from '../utils.js'; export class PlaywrightController extends BrowserController< BrowserType, @@ -36,14 +32,8 @@ export class PlaywrightController extends BrowserController< } protected async _newPage(contextOptions?: SafeParameters[0]): Promise { - if ( - contextOptions !== undefined && - !this.launchContext.useIncognitoPages && - !this.launchContext.experimentalContainers - ) { - throw new Error( - 'A new page can be created with provided context only when using incognito pages or experimental containers.', - ); + if (contextOptions !== undefined && !this.launchContext.useIncognitoPages) { + throw new Error('A new page can be created with provided context only when using incognito pages.'); } let close = async () => {}; @@ -82,50 +72,6 @@ export class PlaywrightController extends BrowserController< await close(); }); - if (this.launchContext.experimentalContainers) { - await page.goto('data:text/plain,tabid'); - await page.waitForNavigation(); - const { tabid, proxyip }: { tabid: number; proxyip: string } = JSON.parse( - decodeURIComponent(page.url().slice('about:blank#'.length)), - ); - - if (contextOptions?.proxy) { - const url = new URL(contextOptions.proxy.server); - url.username = contextOptions.proxy.username ?? ''; - url.password = contextOptions.proxy.password ?? ''; - - (this.browserPlugin as PlaywrightPlugin)._containerProxyServer!.ipToProxy.set(proxyip, url.href); - } - - if (this.browserPlugin.library.name() === 'firefox') { - // Playwright does not support creating new CDP sessions with Firefox - } else { - const session = await page.context().newCDPSession(page); - await session.send('Network.enable'); - - session.on('Network.responseReceived', (responseReceived) => { - const logOnly = ['Document', 'XHR', 'Fetch', 'EventSource', 'WebSocket', 'Other']; - if (!logOnly.includes(responseReceived.type)) { - return; - } - - const { response } = responseReceived; - if (response.fromDiskCache || response.fromPrefetchCache || response.fromServiceWorker) { - return; - } - - const { remoteIPAddress } = response; - if (remoteIPAddress && remoteIPAddress !== proxyip) { - console.warn( - `Request to ${response.url} was through ${remoteIPAddress} instead of ${proxyip}`, - ); - } - }); - } - - tabIds.set(page, tabid); - } - tryCancel(); return page; @@ -147,46 +93,11 @@ export class PlaywrightController extends BrowserController< protected async _getCookies(page: Page): Promise { const context = page.context(); - const cookies = await context.cookies(); - - if (this.launchContext.experimentalContainers) { - const tabId = tabIds.get(page); - - if (tabId === undefined) { - throw new Error('Failed to find tabId for page'); - } - - const key = keyFromTabId(tabId); - - return cookies - .filter((cookie) => cookie.name.startsWith(key)) - .map((cookie) => ({ - ...cookie, - name: cookie.name.slice(key.length), - })); - } - - return cookies; + return context.cookies(); } protected async _setCookies(page: Page, cookies: Cookie[]): Promise { const context = page.context(); - - if (this.launchContext.experimentalContainers) { - const tabId = tabIds.get(page); - - if (tabId === undefined) { - throw new Error('Failed to find tabId for page'); - } - - const key = keyFromTabId(tabId); - - cookies = cookies.map((cookie) => ({ - ...cookie, - name: `${key}${cookie.name}`, - })); - } - return context.addCookies(cookies); } } diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts index 5e59d4656588..6520ecebb6ff 100644 --- a/packages/browser-pool/src/playwright/playwright-plugin.ts +++ b/packages/browser-pool/src/playwright/playwright-plugin.ts @@ -1,37 +1,16 @@ import fs from 'node:fs'; -import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright'; -import type { BrowserController } from '../abstract-classes/browser-controller'; -import { BrowserPlugin } from '../abstract-classes/browser-plugin'; -import { anonymizeProxySugar } from '../anonymize-proxy'; -import { createProxyServerForContainers } from '../container-proxy-server'; -import type { LaunchContext } from '../launch-context'; -import { log } from '../logger'; -import { getLocalProxyAddress } from '../proxy-server'; -import type { SafeParameters } from '../utils'; -import { loadFirefoxAddon } from './load-firefox-addon'; -import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser'; -import { PlaywrightController } from './playwright-controller'; - -const getFreePort = async () => { - return new Promise((resolve, reject) => { - const server = net - .createServer() - .once('error', reject) - .listen(() => { - resolve((server.address() as net.AddressInfo).port); - server.close(); - }); - }); -}; - -// __dirname = browser-pool/dist/playwright -// taacPath = browser-pool/dist/tab-as-a-container -const taacPath = path.join(__dirname, '..', 'tab-as-a-container'); +import { BrowserPlugin } from '../abstract-classes/browser-plugin.js'; +import { anonymizeProxySugar } from '../anonymize-proxy.js'; +import type { createProxyServerForContainers } from '../container-proxy-server.js'; +import type { LaunchContext } from '../launch-context.js'; +import { log } from '../logger.js'; +import { getLocalProxyAddress } from '../proxy-server.js'; +import type { SafeParameters } from '../utils.js'; +import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js'; +import { PlaywrightController } from './playwright-controller.js'; export class PlaywrightPlugin extends BrowserPlugin< BrowserType, @@ -42,10 +21,7 @@ export class PlaywrightPlugin extends BrowserPlugin< _containerProxyServer?: Awaited>; protected async _launch(launchContext: LaunchContext): Promise { - const { launchOptions, useIncognitoPages, proxyUrl } = launchContext; - - let { userDataDir } = launchContext; - + const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext; let browser: PlaywrightBrowser; // Required for the `proxy` context option to work. @@ -79,44 +55,6 @@ export class PlaywrightPlugin extends BrowserPlugin< }); } } else { - const experimentalContainers = launchContext.experimentalContainers && this.library.name() !== 'webkit'; - let firefoxPort: number | undefined; - - if (experimentalContainers) { - launchOptions!.args = [...(launchOptions!.args ?? [])]; - - // Use native headless mode so we can load an extension - if (launchOptions!.headless && this.library.name() === 'chromium') { - launchOptions!.args.push('--headless=chrome'); - } - - if (this.library.name() === 'chromium') { - launchOptions!.args.push( - `--disable-extensions-except=${taacPath}`, - `--load-extension=${taacPath}`, - ); - } else if (this.library.name() === 'firefox') { - firefoxPort = await getFreePort(); - - launchOptions!.args.push(`--start-debugger-server=${firefoxPort}`); - - const prefs = { - 'devtools.debugger.remote-enabled': true, - 'devtools.debugger.prompt-connection': false, - }; - - const prefsRaw = Object.entries(prefs) - .map(([name, value]) => `user_pref(${JSON.stringify(name)}, ${JSON.stringify(value)});`) - .join('\n'); - - if (userDataDir === '') { - userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'apify-playwright-firefox-taac-')); - } - - fs.writeFileSync(path.join(userDataDir, 'user.js'), prefsRaw); - } - } - const browserContext = await this.library .launchPersistentContext(userDataDir, launchOptions) .catch((error) => { @@ -132,34 +70,6 @@ export class PlaywrightPlugin extends BrowserPlugin< } }); - if (experimentalContainers) { - if (this.library.name() === 'firefox') { - const loaded = await loadFirefoxAddon(firefoxPort!, '127.0.0.1', taacPath); - - if (!loaded) { - await browserContext.close(); - throw new Error('Failed to load Firefox experimental containers addon'); - } - } - - // Wait for the extension to load. - const checker = await browserContext.newPage(); - await checker.goto('data:text/plain,tabid'); - await checker.waitForNavigation(); - await checker.close(); - - this._containerProxyServer = await createProxyServerForContainers(); - - const page = await browserContext.newPage(); - await page.goto(`data:text/plain,proxy#{"port":${this._containerProxyServer.port}}`); - await page.waitForNavigation(); - await page.close(); - - browserContext.on('close', async () => { - await this._containerProxyServer!.close(true); - }); - } - if (anonymizedProxyUrl) { browserContext.on('close', async () => { await close(); @@ -199,12 +109,8 @@ export class PlaywrightPlugin extends BrowserPlugin< ); } - protected _createController(): BrowserController< - BrowserType, - SafeParameters[0], - PlaywrightBrowser - > { - return new PlaywrightController(this); + override createController(): PlaywrightController { + return new PlaywrightController(this as any); } protected async _addProxyToLaunchOptions(launchContext: LaunchContext): Promise { diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts index d51a8ef1d514..3c654c99ce71 100644 --- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts +++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts @@ -4,9 +4,9 @@ import type * as PuppeteerTypes from 'puppeteer'; import { tryCancel } from '@apify/timeout'; -import { BrowserController } from '../abstract-classes/browser-controller'; -import { anonymizeProxySugar } from '../anonymize-proxy'; -import { log } from '../logger'; +import { BrowserController } from '../abstract-classes/browser-controller.js'; +import { anonymizeProxySugar } from '../anonymize-proxy.js'; +import { log } from '../logger.js'; export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOptions { proxyUsername?: string; @@ -41,9 +41,7 @@ export class PuppeteerController extends BrowserController< protected async _newPage(contextOptions?: PuppeteerNewPageOptions): Promise { if (contextOptions !== undefined) { if (!this.launchContext.useIncognitoPages) { - throw new Error( - 'A new page can be created with provided context only when using incognito pages or experimental containers.', - ); + throw new Error('A new page can be created with provided context only when using incognito pages.'); } let close = async () => {}; diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts index f273916c8ea9..b8e3b8edafb3 100644 --- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts +++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts @@ -4,14 +4,13 @@ import type { Dictionary } from '@crawlee/types'; import type Puppeteer from 'puppeteer'; import type * as PuppeteerTypes from 'puppeteer'; -import type { BrowserController } from '../abstract-classes/browser-controller'; -import { BrowserPlugin } from '../abstract-classes/browser-plugin'; -import { anonymizeProxySugar } from '../anonymize-proxy'; -import type { LaunchContext } from '../launch-context'; -import { log } from '../logger'; -import { noop } from '../utils'; -import type { PuppeteerNewPageOptions } from './puppeteer-controller'; -import { PuppeteerController } from './puppeteer-controller'; +import { BrowserPlugin } from '../abstract-classes/browser-plugin.js'; +import { anonymizeProxySugar } from '../anonymize-proxy.js'; +import type { LaunchContext } from '../launch-context.js'; +import { log } from '../logger.js'; +import { noop } from '../utils.js'; +import type { PuppeteerNewPageOptions } from './puppeteer-controller.js'; +import { PuppeteerController } from './puppeteer-controller.js'; const PROXY_SERVER_ARG = '--proxy-server='; @@ -39,12 +38,8 @@ export class PuppeteerPlugin extends BrowserPlugin< } catch { // ignore } - const { launchOptions, userDataDir, useIncognitoPages, experimentalContainers, proxyUrl } = launchContext; - - if (experimentalContainers) { - throw new Error('Experimental containers are only available with Playwright'); - } + const { launchOptions, userDataDir, useIncognitoPages, proxyUrl } = launchContext; launchOptions!.userDataDir = launchOptions!.userDataDir ?? userDataDir; if (launchOptions!.headless === false) { @@ -188,12 +183,7 @@ export class PuppeteerPlugin extends BrowserPlugin< return browser; } - protected _createController(): BrowserController< - typeof Puppeteer, - PuppeteerTypes.LaunchOptions, - PuppeteerTypes.Browser, - PuppeteerNewPageOptions - > { + override createController(): PuppeteerController { return new PuppeteerController(this); } diff --git a/packages/browser-pool/src/utils.ts b/packages/browser-pool/src/utils.ts index f6ea9b9d4cc9..ae224fee62e5 100644 --- a/packages/browser-pool/src/utils.ts +++ b/packages/browser-pool/src/utils.ts @@ -1,6 +1,6 @@ -import type { BrowserPlugin } from './abstract-classes/browser-plugin'; -import type { PlaywrightPlugin } from './playwright/playwright-plugin'; -import type { PuppeteerPlugin } from './puppeteer/puppeteer-plugin'; +import type { BrowserPlugin } from './abstract-classes/browser-plugin.js'; +import type { PlaywrightPlugin } from './playwright/playwright-plugin.js'; +import type { PuppeteerPlugin } from './puppeteer/puppeteer-plugin.js'; export type UnwrapPromise = T extends PromiseLike ? UnwrapPromise : T; diff --git a/packages/browser-pool/tab-as-a-container/background.js b/packages/browser-pool/tab-as-a-container/background.js deleted file mode 100644 index f315fcdf1772..000000000000 --- a/packages/browser-pool/tab-as-a-container/background.js +++ /dev/null @@ -1,433 +0,0 @@ -'use strict'; - -/* eslint-disable no-undef */ - -const isFirefox = navigator.userAgent.includes('Firefox'); - -const webRequestPermissions = { - blockingRequest: isFirefox ? ['blocking', 'requestHeaders'] : ['blocking', 'requestHeaders', 'extraHeaders'], - blockingResponse: isFirefox ? ['blocking', 'responseHeaders'] : ['blocking', 'responseHeaders', 'extraHeaders'], -}; - -chrome.privacy.network.networkPredictionEnabled.set({ value: false }); - -const translator = new Map(); -const counter = new Map(); - -const getOpenerId = (id) => { - if (typeof id !== 'number' || !Number.isFinite(id)) { - throw new Error('Expected `id` to be a number'); - } - - if (translator.has(id)) { - const opener = translator.get(id); - - if (translator.has(opener)) { - throw new Error('Opener is not the most ascendent'); - } - - // console.log(`getopener ${id} -> ${opener}`); - return opener; - } - - return id; -}; - -const keyFromTabId = (tabId) => `.${tabId}.`; - -const getCookieURL = (cookie) => { - const protocol = cookie.secure ? 'https:' : 'http:'; - const fixedDomain = cookie.domain[0] === '.' ? cookie.domain.slice(1) : cookie.domain; - const url = `${protocol}//${fixedDomain}${cookie.path}`; - - return url; -}; - -// Rewrite cookies that were programmatically set to tabId instead of openerId. -// This is required because we cannot reliably get openerId inside Playwright. -chrome.cookies.onChanged.addListener(async (changeInfo) => { - if (!changeInfo.removed) { - const { cookie } = changeInfo; - - if (cookie.name[0] !== '.') { - return; - } - - const dotIndex = cookie.name.indexOf('.', 1); - if (dotIndex === -1) { - return; - } - - const tabId = Number(cookie.name.slice(1, dotIndex)); - - if (!Number.isFinite(tabId)) { - return; - } - - const realCookieName = cookie.name.slice(dotIndex + 1); - const opener = getOpenerId(tabId); - - if (tabId !== opener) { - console.log(`${realCookieName} -> ${keyFromTabId(opener)}`); - - await chrome.cookies.remove({ - name: cookie.name, - url: getCookieURL(cookie), - storeId: cookie.storeId, - }); - - delete cookie.hostOnly; - delete cookie.session; - - await chrome.cookies.set({ - ...cookie, - name: `${keyFromTabId(opener)}${realCookieName}`, - url: getCookieURL(cookie), - }); - } - } -}); - -chrome.webRequest.onBeforeSendHeaders.addListener( - (details) => { - for (const header of details.requestHeaders) { - if (header.name.toLowerCase() === 'cookie') { - const id = keyFromTabId(getOpenerId(details.tabId)); - - const fixedCookies = header.value - .split('; ') - .filter((x) => x.startsWith(id)) - .map((x) => x.slice(id.length)) - .join('; '); - header.value = fixedCookies; - } - - // Sometimes Chrome makes a request on a ghost tab. - // We don't want these in order to prevent cluttering cookies. - // Yes, `webNavigation.onCommitted` is emitted and `webNavigation.onCreatedNavigationTarget` is not. - if (header.name.toLowerCase() === 'purpose' && header.value === 'prefetch' && !counter.has(details.tabId)) { - console.log(details); - return { - cancel: true, - }; - } - - // This one is for Firefox - if (header.name.toLowerCase() === 'x-moz' && header.value === 'prefetch' && !counter.has(details.tabId)) { - console.log(details); - return { - cancel: true, - }; - } - - if (['beacon', 'csp_report', 'ping', 'speculative'].includes(details.type)) { - console.log(details); - return { - cancel: true, - }; - } - - if (details.tabId === -1) { - console.log(details); - } - } - - return { - requestHeaders: details.requestHeaders.filter( - (header) => header.name.toLowerCase() !== 'cookie' || header.value !== '', - ), - }; - }, - { urls: [''] }, - webRequestPermissions.blockingRequest, -); - -// Firefox Bug: doesn't catch https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-uri -chrome.webRequest.onHeadersReceived.addListener( - (details) => { - for (const header of details.responseHeaders) { - if (header.name.toLowerCase() === 'set-cookie') { - const parts = header.value.split('\n'); - - // `details.tabId` === -1 when Chrome is making internal requests, such downloading a service worker. - - const openerId = getOpenerId(details.tabId); - - header.value = parts - .map((part) => { - const equalsIndex = part.indexOf('='); - if (equalsIndex === -1) { - return `${keyFromTabId(openerId)}=${part.trimStart()}`; - } - return keyFromTabId(openerId) + part.trimStart(); - }) - .join('\n'); - } - } - - return { - responseHeaders: details.responseHeaders, - }; - }, - { urls: [''] }, - webRequestPermissions.blockingResponse, -); - -chrome.tabs.onRemoved.addListener(async (tabId) => { - const opener = getOpenerId(tabId); - translator.delete(tabId); - - if (counter.has(opener)) { - counter.set(opener, counter.get(opener) - 1); - - if (counter.get(opener) < 1) { - counter.delete(opener); - } else { - return; - } - } - - const id = keyFromTabId(opener); - - chrome.cookies.getAll({}, async (cookies) => { - await Promise.allSettled( - cookies - .filter((cookie) => cookie.name.startsWith(id)) - .map((cookie) => { - return chrome.cookies.remove({ - name: cookie.name, - url: getCookieURL(cookie), - storeId: cookie.storeId, - }); - }), - ); - }); -}); - -// Proxy per tab -const getProxyConfiguration = (scheme, host, port) => { - return { - mode: 'fixed_servers', - rules: { - proxyForHttp: { - scheme, - host, - port, - }, - proxyForHttps: { - scheme, - host, - port, - }, - }, - }; -}; - -const localhostIpCache = new Map(); -const localHostIp = [127, 0, 0, 1]; -const getNextLocalhostIp = (openerId) => { - if (localhostIpCache.has(openerId)) { - return localhostIpCache.get(openerId); - } - - const result = localHostIp.join('.'); - - localhostIpCache.set(openerId, result); - - if (localHostIp[3] === 254) { - if (localHostIp[2] === 255) { - if (localHostIp[1] === 255) { - localHostIp[1] = 0; - } else { - localHostIp[1]++; - } - - localHostIp[2] = 0; - } else { - localHostIp[2]++; - } - - localHostIp[3] = 1; - } else { - localHostIp[3]++; - } - - // [127.0.0.1 - 127.255.255.254] = 1 * 255 * 255 * 254 = 16 516 350 - while (localhostIpCache.length >= 1 * 255 * 255 * 254) { - localhostIpCache.delete(localhostIpCache.keys().next().value); - } - - return result; -}; - -let proxyPort; - -// Clear extension's proxy settings on reload -if (isFirefox) { - browser.proxy.settings.clear({}); -} else { - chrome.proxy.settings.clear({}); -} - -// Proxy per tab -if (isFirefox) { - // On Firefox, we could use the `dns` permission to enforce DoH - // but then the extension would not be compatible with Chrome. - // Therefore users need to manually set the DNS settings. - - browser.proxy.onRequest.addListener( - (details) => { - const openerId = getOpenerId(details.tabId); - - if (typeof proxyPort === 'number') { - return { - type: 'http', - host: getNextLocalhostIp(openerId), - port: proxyPort, - }; - } - return { - type: 'direct', - }; - }, - { urls: [''] }, - ); -} else { - // The connection is not yet created with `onBeforeSendHeaders`, but is with `onSendHeaders`. - chrome.webRequest.onBeforeSendHeaders.addListener( - (details) => { - const openerId = getOpenerId(details.tabId); - - if (typeof proxyPort === 'number') { - chrome.proxy.settings.set({ - value: getProxyConfiguration('http', getNextLocalhostIp(openerId), proxyPort), - scope: 'regular', - }); - } else { - chrome.proxy.settings.clear({}); - } - }, - { urls: [''] }, - webRequestPermissions.blockingRequest, - ); -} - -// External communication. Note: the JSON keys are lowercased by the browser. -const routes = Object.assign(Object.create(null), { - async tabid(details) { - return { tabid: details.tabId, proxyip: getNextLocalhostIp(details.tabId) }; - }, - async proxy(details, body) { - proxyPort = body.port; - - return ''; - }, -}); - -const onCompleted = async (details) => { - const textPlain = 'data:text/plain,'; - - if (details.frameId === 0 && details.url.startsWith(textPlain)) { - try { - const url = new URL(details.url); - const route = url.pathname.slice('text/plain,'.length); - - if (route in routes) { - const hash = url.hash.slice(1); - - let body = {}; - - if (hash !== '') { - try { - body = JSON.parse(decodeURIComponent(hash)); - } catch { - // Empty on purpose. - } - } - - // Different protocols are required, otherwise `onCompleted` won't be emitted. - const result = await routes[route](details, body); - if (result !== undefined) { - await chrome.tabs.update(details.tabId, { - url: `about:blank#${encodeURIComponent(JSON.stringify(result))}`, - }); - } - } - } catch { - // Invalid URL, ignore. - } - } -}; - -chrome.webNavigation.onCompleted.addListener(onCompleted); - -// Load content scripts. -void (async () => { - const contentResponse = await fetch(chrome.runtime.getURL('content.js')); - const contentText = await contentResponse.text(); - - // `tabs.onCreated` doesn't work here when manually creating new tabs, - // because the opener is the current tab active. - // - // This events only fires when the page opens something. - chrome.webNavigation.onCreatedNavigationTarget.addListener((details) => { - translator.set(details.tabId, getOpenerId(details.sourceTabId)); - - const opener = getOpenerId(details.tabId); - - if (counter.has(opener)) { - counter.set(opener, counter.get(opener) + 1); - } else { - counter.set(opener, 2); // the current one + opener = 2 - } - }); - - chrome.webNavigation.onCommitted.addListener(async (details) => { - if (details.url.startsWith('chrome')) { - return; - } - - const executeCodeInPageContext = ` - const script = document.createElement('script'); - script.textContent = code; - - const destination = document.head ?? document.documentElement; - - if (document instanceof HTMLDocument) { - destination.append(script); - script.remove(); - } - `; - - // Race condition: website scripts may run first - await chrome.tabs.executeScript(details.tabId, { - code: `'use strict'; - (() => { - if (window.totallyRandomString) { - return; - } - - window.totallyRandomString = true; - - const code = "'use strict'; const tabId = '${getOpenerId( - details.tabId, - )}'; (() => {\\n" + ${JSON.stringify(contentText)} + "\\n})();\\n"; - ${executeCodeInPageContext} - })(); - `, - matchAboutBlank: true, - allFrames: true, - runAt: 'document_start', - }); - }); - - chrome.tabs.query({}, async (tabs) => { - for (const tab of tabs) { - await onCompleted({ - frameId: 0, - url: tab.url, - tabId: tab.id, - }); - } - }); -})(); diff --git a/packages/browser-pool/tab-as-a-container/content.js b/packages/browser-pool/tab-as-a-container/content.js deleted file mode 100644 index efbbff7c0835..000000000000 --- a/packages/browser-pool/tab-as-a-container/content.js +++ /dev/null @@ -1,611 +0,0 @@ -// When in doubt, refer to https://github.com/nodejs/node/blob/main/doc/contributing/primordials.md - -/* eslint-disable no-undef */ -/* eslint-disable no-cond-assign */ -/* eslint-disable prefer-rest-params */ -/* eslint-disable no-shadow */ - -// TODO: https://developer.mozilla.org/en-US/docs/Web/API/Cookie_Store_API -// TODO: custom error messages for Firefox (for now it uses Chrome's) - -// The only way to detect this "container" is to benchmark document.cookie or compare localStorage performance with sessionStorage (it's the same). - -const isFirefox = navigator.userAgent.includes('Firefox'); -const tabPrefix = `.${tabId}.`; - -const { - String, - Array, - Set, - TypeError, - WeakMap, - Object, - Number, - Function, - Proxy, - IDBFactory, - IDBDatabase, - BroadcastChannel, - Storage, - // We don't have to implement StorageEvent because this implementation does not use localStorage at all. -} = globalThis; - -const ObjectDefineProperty = Object.defineProperty; -const ObjectDefineProperties = Object.defineProperties; -const ObjectGetOwnPropertyDescriptors = Object.getOwnPropertyDescriptors; -const ObjectGetPrototypeOf = Object.getPrototypeOf; -const ObjectGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; -const ObjectCreate = Object.create; -const ObjectEntries = Object.entries; -const ReflectGet = Reflect.get; -const ReflectSet = Reflect.set; -const ObjectKeys = Object.keys; -const NumberIsFinite = Number.isFinite; - -const clonePrototype = (from) => { - const target = ObjectCreate(null); - const prototype = ObjectGetOwnPropertyDescriptors(from.prototype); - - const entries = ObjectEntries(prototype); - - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - - const { 0: name, 1: descriptor } = entry; - target[name] = ObjectCreate(null); - - if ('get' in descriptor) { - target[name].get = descriptor.get; - } - - if ('set' in descriptor) { - target[name].set = descriptor.set; - } - - if ('value' in descriptor) { - target[name] = descriptor.value; - } - } - - return target; -}; - -const StringSplitSafe = (string, separator) => { - const result = []; - const separatorLength = separator.length; - - if (separatorLength === 0) { - throw new Error('Separator must not be empty'); - } - - let startFrom = 0; - let index; - while ((index = StringPrototype.indexOf.call(string, separator, startFrom)) !== -1) { - ArrayPrototype.push.call(result, StringPrototype.slice.call(string, startFrom, index)); - - startFrom = index + separatorLength; - } - - const lastChunk = StringPrototype.slice.call(string, startFrom); - - ArrayPrototype.push.call(result, lastChunk); - - return result; -}; - -const fixStack = (error) => { - const lines = StringSplitSafe(error.stack, '\n'); - - if (isFirefox) { - ArrayPrototype.splice.call(lines, 0, 1); - } else { - ArrayPrototype.splice.call(lines, 1, 1); - } - - error.stack = ArrayPrototype.join.call(lines, '\n'); - - return error; -}; - -const SetPrototype = clonePrototype(Set); -const WeakMapPrototype = clonePrototype(WeakMap); -const ArrayPrototype = clonePrototype(Array); -const StringPrototype = clonePrototype(String); -const IDBFactoryPrototype = clonePrototype(IDBFactory); -const IDBDatabasePrototype = clonePrototype(IDBDatabase); -const StoragePrototype = clonePrototype(Storage); - -const privates = new WeakMap(); - -let invocable = false; - -const FakeStorage = class Storage { - constructor() { - if (invocable) { - throw fixStack(new TypeError('Illegal constructor')); - } - - WeakMapPrototype.set.call(privates, this, arguments[0]); - } - - get length() { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - const { storage, prefix } = priv; - const length = StoragePrototype.length.get.call(storage); - - let fakeLength = 0; - for (let i = 0; i < length; i++) { - const storageKey = StoragePrototype.key.call(storage, i); - if (StringPrototype.startsWith.call(storageKey, prefix)) { - fakeLength++; - } - } - - return fakeLength; - } - - clear() { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - const { storage, prefix } = priv; - const length = StoragePrototype.length.get.call(storage); - const keys = []; - - for (let i = 0; i < length; i++) { - ArrayPrototype.push.call(keys, StoragePrototype.key.call(storage, i)); - } - - for (let i = 0; i < length; i++) { - const storageKey = keys[i]; - if (StringPrototype.startsWith.call(storageKey, prefix)) { - StoragePrototype.removeItem.call(storage, storageKey); - } - } - } - - key(index) { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - if (arguments.length === 0) { - throw fixStack( - new TypeError(`Failed to execute 'key' on 'Storage': 1 argument required, but only 0 present.`), - ); - } - - index = NumberIsFinite(index) ? index : 0; - - const { storage, prefix } = priv; - const length = StoragePrototype.length.get.call(storage); - - let fakeLength = 0; - for (let i = 0; i < length; i++) { - const storageKey = StoragePrototype.key.call(storage, i); - - if (StringPrototype.startsWith.call(storageKey, prefix)) { - if (fakeLength === index) { - return StringPrototype.slice.call(storageKey, prefix.length); - } - - fakeLength++; - } - } - - return null; - } - - getItem(key) { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - if (arguments.length === 0) { - throw fixStack( - new TypeError(`Failed to execute 'getItem' on 'Storage': 1 argument required, but only 0 present.`), - ); - } - - return StoragePrototype.getItem.call(priv.storage, priv.prefix + key); - } - - removeItem(key) { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - if (arguments.length === 0) { - throw fixStack( - new TypeError(`Failed to execute 'removeItem' on 'Storage': 1 argument required, but only 0 present.`), - ); - } - - StoragePrototype.removeItem.call(priv.storage, priv.prefix + key); - } - - setItem(key, value) { - const priv = WeakMapPrototype.get.call(privates, this); - if (!priv) { - throw fixStack(new TypeError('Illegal invocation')); - } - - if (arguments.length === 0 || arguments.length === 1) { - throw fixStack( - new TypeError( - `Failed to execute 'setItem' on 'Storage': 2 arguments required, but only ${arguments.length} present.`, - ), - ); - } - - StoragePrototype.setItem.call(priv.storage, priv.prefix + key, value); - } -}; - -const FakeStoragePrototype = clonePrototype(FakeStorage); - -const createStorage = ({ storage, prefix }) => { - invocable = false; - const fake = new FakeStorage({ storage, prefix }); - invocable = true; - - const proxy = new Proxy(fake, { - __proto__: null, - // Default: - // apply: (target, thisArg, args) => {}, - // construct(target, args) => {}, - // setPrototypeOf: (target, proto) => {}, - // getPrototypeOf: (target) => {}, - defineProperty: (target, key, descriptor) => { - if ('set' in descriptor || 'get' in descriptor) { - throw fixStack( - new TypeError(`Failed to set a named property on 'Storage': Accessor properties are not allowed.`), - ); - } - - FakeStoragePrototype.setItem.call(target, key, descriptor.value); - }, - deleteProperty: (target, key) => { - if (typeof key === 'symbol') { - delete target[key]; - } else { - FakeStoragePrototype.removeItem.call(target, key); - } - - return true; - }, - get: (target, key) => { - if (typeof key === 'symbol') { - return target[key]; - } - - if (key in target) { - return ReflectGet(target, key); - } - - return FakeStoragePrototype.getItem.call(target, key) ?? undefined; - }, - set: (target, key, value) => { - if (typeof key === 'symbol') { - ObjectDefineProperty(target, key, { - __proto__: null, - value, - configurable: true, - writable: true, - enumerable: false, - }); - - return true; - } - - if (key in target) { - return ReflectSet(target, key, value); - } - - return FakeStoragePrototype.setItem.call(target, key, value) ?? true; - }, - has: (target, key) => { - if (key in target) { - return true; - } - - return FakeStoragePrototype.getItem.call(target, key) !== null; - }, - isExtensible: () => { - return true; - }, - preventExtensions: () => { - throw fixStack(new TypeError(`Cannot prevent extensions`)); - }, - getOwnPropertyDescriptor: (target, key) => { - if (key in target) { - return ObjectGetOwnPropertyDescriptor(ObjectGetPrototypeOf(target), key); - } - - const value = FakeStoragePrototype.getItem.call(target, key); - - if (value !== null) { - return { - value, - writable: true, - enumerable: true, - configurable: true, - }; - } - }, - ownKeys: (target) => { - const keys = []; - - const { storage, prefix } = WeakMapPrototype.get.call(privates, target); - const length = StoragePrototype.length.get.call(storage); - - for (let i = 0; i < length; i++) { - const storageKey = StoragePrototype.key.call(storage, i); - - if (StringPrototype.startsWith.call(storageKey, prefix)) { - ArrayPrototype.push.call(keys, StringPrototype.slice.call(storageKey, prefix.length)); - } - } - - ArrayPrototype.push.apply(keys, ObjectKeys(target)); - - const set = new Set(); - - for (let i = 0; i < keys.length; i++) { - SetPrototype.add.call(set, keys[i]); - } - - return ArrayPrototype.slice.call(set); - }, - }); - - privates.set(proxy, privates.get(fake)); - - return proxy; -}; - -const toHide = new WeakMap(); -for (const Type of [Function, Object, Array]) { - const create = (fallback) => - function () { - if (this instanceof FakeStorage) { - return '[object Storage]'; - } - - if (WeakMapPrototype.has.call(toHide, this)) { - return `function ${WeakMapPrototype.get.call(toHide, this)}() { [native code] }`; - } - - return fallback.call(this); - }; - - const toString = create(Type.prototype.toString); - const toLocaleString = create(Type.prototype.toLocaleString); - - WeakMapPrototype.set.call(toHide, toString, 'toString'); - WeakMapPrototype.set.call(toHide, toLocaleString, 'toLocaleString'); - - Object.defineProperty(Type.prototype, 'toString', { - __proto__: null, - value: toString, - }); - Object.defineProperty(Type.prototype, 'toLocaleString', { - __proto__: null, - value: toLocaleString, - }); -} - -// https://stackoverflow.com/q/30481516 -try { - // We use sessionStorage as the underlying storage for localStorage. - // This way we do not have to worry about clean up. - const { sessionStorage } = globalThis; - - const fakeLocalStorage = createStorage({ storage: sessionStorage, prefix: 'l.' }); - const fakeSessionStorage = createStorage({ storage: sessionStorage, prefix: 's.' }); - - const getLocalStorage = function localStorage() { - return fakeLocalStorage; - }; - const getSessionStorage = function sessionStorage() { - return fakeSessionStorage; - }; - - WeakMapPrototype.set.call(toHide, FakeStorage, 'Storage'); - WeakMapPrototype.set.call(toHide, FakeStoragePrototype.key, 'key'); - WeakMapPrototype.set.call(toHide, FakeStoragePrototype.getItem, 'getItem'); - WeakMapPrototype.set.call(toHide, FakeStoragePrototype.setItem, 'setItem'); - WeakMapPrototype.set.call(toHide, FakeStoragePrototype.removeItem, 'removeItem'); - WeakMapPrototype.set.call(toHide, FakeStoragePrototype.clear, 'clear'); - WeakMapPrototype.set.call(toHide, getLocalStorage, 'get localStorage'); - WeakMapPrototype.set.call(toHide, getSessionStorage, 'get sessionStorage'); - - ObjectDefineProperties(window, { - __proto__: null, - Storage: { - __proto__: null, - value: FakeStorage, - configurable: true, - enumerable: false, - writable: true, - }, - localStorage: { - __proto__: null, - configurable: true, - enumerable: true, - get: getLocalStorage, - set: undefined, - }, - sessionStorage: { - __proto__: null, - configurable: true, - enumerable: true, - get: getSessionStorage, - set: undefined, - }, - }); -} catch (error) { - console.error(error); -} - -{ - const { Document } = globalThis; - - const realGetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').get; - const realSetCookie = ObjectGetOwnPropertyDescriptor(Document.prototype, 'cookie').set; - - const getCookie = function cookie() { - try { - const cookies = StringSplitSafe(realGetCookie.call(this), '; '); - const filtered = ArrayPrototype.filter.call(cookies, (cookie) => - StringPrototype.startsWith.call(cookie, tabPrefix), - ); - const mapped = ArrayPrototype.map.call(filtered, (cookie) => { - const result = StringPrototype.slice.call(cookie, tabPrefix.length); - - if (result[0] === '=') { - return StringPrototype.slice.call(result, 1); - } - - return result; - }); - - return ArrayPrototype.join.call(mapped, '; '); - } catch (error) { - throw fixStack(error); - } - }; - - const setCookie = function cookie(cookieString) { - cookieString = StringPrototype.trimStart.call(String(cookieString)); - - const delimiterIndex = StringPrototype.indexOf.call(cookieString, ';'); - const equalsIndex = StringPrototype.indexOf.call(cookieString, '='); - if (equalsIndex === -1 || (delimiterIndex !== -1 && equalsIndex > delimiterIndex)) { - cookieString = `=${cookieString}`; - } - - try { - realSetCookie.call(this, tabPrefix + cookieString); - } catch (error) { - throw fixStack(error); - } - }; - - WeakMapPrototype.set.call(toHide, getCookie, 'get cookie'); - WeakMapPrototype.set.call(toHide, setCookie, 'set cookie'); - - ObjectDefineProperty(Document.prototype, 'cookie', { - __proto__: null, - configurable: true, - enumerable: true, - get: getCookie, - set: setCookie, - }); -} - -{ - const openDatabase = function open(name) { - try { - return IDBFactoryPrototype.open.call(this, tabPrefix + name); - } catch (error) { - throw fixStack(error); - } - }; - - const deleteDatabase = function deleteDatabase(name) { - try { - return IDBFactoryPrototype.deleteDatabase.call(this, tabPrefix + name); - } catch (error) { - throw fixStack(error); - } - }; - - const databaseName = function name() { - try { - return StringPrototype.slice.call(IDBDatabasePrototype.name.get.call(this), tabPrefix.length); - } catch (error) { - throw fixStack(error); - } - }; - - WeakMapPrototype.set.call(toHide, openDatabase, 'open'); - WeakMapPrototype.set.call(toHide, deleteDatabase, 'deleteDatabase'); - WeakMapPrototype.set.call(toHide, databaseName, 'get name'); - - ObjectDefineProperties(IDBFactory.prototype, { - __proto__: null, - open: { - __proto__: null, - writable: true, - configurable: true, - enumerable: true, - value: openDatabase, - }, - deleteDatabase: { - __proto__: null, - writable: true, - configurable: true, - enumerable: true, - value: deleteDatabase, - }, - name: { - __proto__: null, - configurable: true, - enumerable: true, - get: databaseName, - set: undefined, - }, - }); -} - -{ - ObjectDefineProperty(window, 'BroadcastChannel', { - __proto__: null, - configurable: true, - enumerable: false, - writable: true, - value: new Proxy(BroadcastChannel, { - __proto__: null, - construct: (Target, name) => { - return new Target(tabPrefix + name); - }, - }), - }); - - WeakMapPrototype.set.call(toHide, window.BroadcastChannel, 'BroadcastChannel'); - - const getBroadcastChannelName = ObjectGetOwnPropertyDescriptor(BroadcastChannel.prototype, 'name').get; - const broadcastChannelName = function name() { - try { - const realName = getBroadcastChannelName.call(this); - - if (StringPrototype.startsWith.call(realName, tabPrefix)) { - return StringPrototype.slice.call(realName, tabPrefix.length); - } - - return realName; - } catch (error) { - throw fixStack(error); - } - }; - - WeakMapPrototype.set.call(toHide, broadcastChannelName, 'get name'); - - ObjectDefineProperty(BroadcastChannel.prototype, 'name', { - __proto__: null, - configurable: true, - enumerable: true, - get: broadcastChannelName, - set: undefined, - }); -} diff --git a/packages/browser-pool/tab-as-a-container/manifest.json b/packages/browser-pool/tab-as-a-container/manifest.json deleted file mode 100644 index cc77a982f9a9..000000000000 --- a/packages/browser-pool/tab-as-a-container/manifest.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "manifest_version": 2, - "name": "Tab as a Container", - "version": "1.0.0", - "background": { - "scripts": ["background.js"], - "persistent": true - }, - "permissions": [ - "webRequest", - "webRequestBlocking", - "webNavigation", - "tabs", - "cookies", - "privacy", - "proxy", - "" - ], - "web_accessible_resources": ["content.js"], - "incognito": "not_allowed" -} diff --git a/packages/browser-pool/test/changing-page-options.test.ts b/packages/browser-pool/test/changing-page-options.test.ts index 69c68953b6fa..f843370f401d 100644 --- a/packages/browser-pool/test/changing-page-options.test.ts +++ b/packages/browser-pool/test/changing-page-options.test.ts @@ -8,7 +8,7 @@ import playwright from 'playwright'; import type { Server as ProxyChainServer } from 'proxy-chain'; import puppeteer from 'puppeteer'; -import { createProxyServer } from '../../../test/browser-pool/browser-plugins/create-proxy-server'; +import { createProxyServer } from '../../../test/browser-pool/browser-plugins/create-proxy-server.js'; describe.each([ ['Puppeteer', new PuppeteerPlugin(puppeteer, { useIncognitoPages: true })], diff --git a/packages/browser-pool/test/proxy-sugar.test.ts b/packages/browser-pool/test/proxy-sugar.test.ts index a16b960cf80b..130ccfb0389e 100644 --- a/packages/browser-pool/test/proxy-sugar.test.ts +++ b/packages/browser-pool/test/proxy-sugar.test.ts @@ -7,7 +7,7 @@ import playwright from 'playwright'; import type { Server as ProxyChainServer } from 'proxy-chain'; import puppeteer from 'puppeteer'; -import { createProxyServer } from '../../../test/browser-pool/browser-plugins/create-proxy-server'; +import { createProxyServer } from '../../../test/browser-pool/browser-plugins/create-proxy-server.js'; describe.each([ ['Puppeteer', new PuppeteerPlugin(puppeteer, { useIncognitoPages: true })], diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json index b06dc1b9ee23..8ddfdcdc3e5f 100644 --- a/packages/cheerio-crawler/package.json +++ b/packages/cheerio-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/cheerio", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,18 +40,18 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@crawlee/http": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "cheerio": "1.0.0-rc.12", - "htmlparser2": "^9.0.0", - "tslib": "^2.4.0" + "@crawlee/http": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "cheerio": "^1.0.0", + "htmlparser2": "^10.0.0", + "tslib": "^2.8.1" } } diff --git a/packages/cheerio-crawler/src/index.ts b/packages/cheerio-crawler/src/index.ts index f4c05bc080f8..adb102844a61 100644 --- a/packages/cheerio-crawler/src/index.ts +++ b/packages/cheerio-crawler/src/index.ts @@ -1,2 +1,2 @@ export * from '@crawlee/http'; -export * from './internals/cheerio-crawler'; +export * from './internals/cheerio-crawler.js'; diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts index 4ed785497282..0450a006d262 100644 --- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts +++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts @@ -1,6 +1,3 @@ -import type { IncomingMessage } from 'node:http'; -import { text as readStreamToString } from 'node:stream/consumers'; - import type { BasicCrawlingContext, Configuration, @@ -16,12 +13,11 @@ import type { SkippedRequestCallback, } from '@crawlee/http'; import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router } from '@crawlee/http'; -import type { Dictionary } from '@crawlee/types'; +import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types'; import { type CheerioRoot, extractUrlsFromCheerio, type RobotsTxtFile } from '@crawlee/utils'; -import type { CheerioOptions } from 'cheerio'; +import type { CheerioAPI, CheerioOptions } from 'cheerio'; import * as cheerio from 'cheerio'; -import { DomHandler, parseDocument } from 'htmlparser2'; -import { WritableStream } from 'htmlparser2/lib/WritableStream'; +import { parseDocument } from 'htmlparser2'; export type CheerioErrorHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler @@ -29,9 +25,11 @@ export type CheerioErrorHandler< > = ErrorHandler>; export interface CheerioCrawlerOptions< + ContextExtension = Dictionary, + ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends HttpCrawlerOptions> {} +> extends HttpCrawlerOptions, ContextExtension, ExtendedContext> {} export type CheerioHook< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler @@ -41,7 +39,12 @@ export type CheerioHook< export interface CheerioCrawlingContext< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends InternalHttpCrawlingContext { +> extends InternalHttpCrawlingContext { + /** + * The raw HTML content of the web page as a string. + */ + body: string; + /** * The [Cheerio](https://cheerio.js.org/) object with parsed HTML. * Cheerio is available only for HTML and XML content types. @@ -77,6 +80,11 @@ export interface CheerioCrawlingContext< * ``` */ parseWithCheerio(selector?: string, timeoutMs?: number): Promise; + + /** + * Helper function for extracting URLs from the parsed HTML and adding them to the request queue. + */ + enqueueLinks(options?: EnqueueLinksOptions): Promise; } export type CheerioRequestHandler< @@ -110,11 +118,11 @@ export type CheerioRequestHandler< * * The crawler finishes when there are no more {@apilink Request} objects to crawl. * - * We can use the `preNavigationHooks` to adjust `gotOptions`: + * We can use the `preNavigationHooks` to adjust the crawling context before the request is made: * * ``` * preNavigationHooks: [ - * (crawlingContext, gotOptions) => { + * (crawlingContext) => { * // ... * }, * ] @@ -161,83 +169,73 @@ export type CheerioRequestHandler< * ``` * @category Crawlers */ -export class CheerioCrawler extends HttpCrawler { +export class CheerioCrawler< + ContextExtension = Dictionary, + ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, +> extends HttpCrawler { /** * All `CheerioCrawler` parameters are passed via an options object. */ - // eslint-disable-next-line @typescript-eslint/no-useless-constructor - constructor(options?: CheerioCrawlerOptions, config?: Configuration) { - super(options, config); + constructor(options?: CheerioCrawlerOptions, config?: Configuration) { + super( + { + ...options, + contextPipelineBuilder: () => + this.buildContextPipeline() + .compose({ + action: async (context) => await this.parseContent(context), + }) + .compose({ action: async (context) => await this.addHelpers(context) }), + }, + config, + ); } - protected override async _parseHTML( - response: IncomingMessage, - isXml: boolean, - crawlingContext: CheerioCrawlingContext, - ) { - const body = await readStreamToString(response); + private async parseContent(crawlingContext: InternalHttpCrawlingContext) { + const isXml = crawlingContext.contentType.type.includes('xml'); + const body = Buffer.isBuffer(crawlingContext.body) + ? crawlingContext.body.toString(crawlingContext.contentType.encoding) + : crawlingContext.body; const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml }); - - const $ = cheerio.load(body, { - xmlMode: isXml, - // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2 - // and not good for scraping. It also does not have a great streaming interface. - // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors. - _useHtmlParser2: true, + const $ = cheerio.load(dom, { + xml: { decodeEntities: true, xmlMode: isXml }, } as CheerioOptions); - const originalEnqueueLinks = crawlingContext.enqueueLinks; - return { - dom, $, body, + }; + } + + private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { $: CheerioAPI }) { + const originalEnqueueLinks = crawlingContext.enqueueLinks; + + return { enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => { - return cheerioCrawlerEnqueueLinks({ + return (await cheerioCrawlerEnqueueLinks({ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) }, - $, + $: crawlingContext.$, requestQueue: await this.getRequestQueue(), robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url), onSkippedRequest: this.handleSkippedRequest, originalRequestUrl: crawlingContext.request.url, finalRequestUrl: crawlingContext.request.loadedUrl, enqueueLinks: originalEnqueueLinks, - }); + })) as BatchAddRequestsResult; // TODO make this type safe }, - }; - } - - // TODO: unused code - remove in 4.0 - protected async _parseHtmlToDom(response: IncomingMessage, isXml: boolean) { - return new Promise((resolve, reject) => { - const domHandler = new DomHandler( - (err, dom) => { - if (err) reject(err); - else resolve(dom); - }, - { xmlMode: isXml }, - ); - const parser = new WritableStream(domHandler, { decodeEntities: true, xmlMode: isXml }); - parser.on('error', reject); - response.on('error', reject).pipe(parser); - }); - } - - protected override async _runRequestHandler(context: CheerioCrawlingContext) { - context.waitForSelector = async (selector?: string, _timeoutMs?: number) => { - if (context.$(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } - }; - context.parseWithCheerio = async (selector?: string, timeoutMs?: number) => { - if (selector) { - await context.waitForSelector(selector, timeoutMs); - } + waitForSelector: async (selector: string, _timeoutMs?: number) => { + if (crawlingContext.$(selector).get().length === 0) { + throw new Error(`Selector '${selector}' not found.`); + } + }, + parseWithCheerio: async (selector?: string, timeoutMs?: number) => { + if (selector) { + await crawlingContext.waitForSelector(selector, timeoutMs); + } - return context.$; + return crawlingContext.$; + }, }; - - await super._runRequestHandler(context); } } diff --git a/packages/cheerio-crawler/test/migration.test.ts b/packages/cheerio-crawler/test/migration.test.ts deleted file mode 100644 index ce0698a82f62..000000000000 --- a/packages/cheerio-crawler/test/migration.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import type { Log } from '@apify/log'; -import log from '@apify/log'; - -import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator'; -import { CheerioCrawler, RequestList } from '../src/index'; - -const localStorageEmulator = new MemoryStorageEmulator(); - -beforeEach(async () => { - await localStorageEmulator.init(); -}); - -afterAll(async () => { - await localStorageEmulator.destroy(); -}); - -describe('Moving from handleRequest* to requestHandler*', () => { - let requestList: RequestList; - let testLogger: Log; - - beforeEach(async () => { - requestList = await RequestList.open(null, []); - testLogger = log.child({ prefix: 'CheerioCrawler' }); - }); - - describe('handlePageFunction -> requestHandler', () => { - it('should log when providing both handlePageFunction and requestHandler', () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - requestHandler: newHandler, - handlePageFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`, - `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `As such, "requestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(newHandler); - }); - - it('should log when providing only the deprecated handlePageFunction', () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - handlePageFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(oldHandler); - }); - - it('should not log when providing only requestHandler', () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - requestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['requestHandler']).toBe(handler); - }); - }); - - describe('handleFailedRequestFunction -> failedRequestHandler', () => { - it('should log when providing both handleFailedRequestFunction and failedRequestHandler', () => { - const oldHandler = () => {}; - const newHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - failedRequestHandler: newHandler, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`, - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `As such, "failedRequestHandler" will be used instead.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(newHandler); - }); - - it('should log when providing only the deprecated handleFailedRequestFunction', () => { - const oldHandler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - handleFailedRequestFunction: oldHandler, - }); - - expect(warningSpy).toHaveBeenCalledWith<[string]>( - [ - `"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`, - `The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`, - ].join('\n'), - ); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(oldHandler); - }); - - it('should not log when providing only failedRequestHandler', () => { - const handler = () => {}; - const warningSpy = vitest.spyOn(testLogger, 'warning'); - - const crawler = new CheerioCrawler({ - requestList, - log: testLogger, - requestHandler: () => {}, - failedRequestHandler: handler, - }); - - expect(warningSpy).not.toHaveBeenCalled(); - - // eslint-disable-next-line dot-notation -- accessing private property - expect(crawler['failedRequestHandler']).toBe(handler); - }); - }); -}); diff --git a/packages/cheerio-crawler/test/xml.test.ts b/packages/cheerio-crawler/test/xml.test.ts index c617ceb0fb3f..b6e84fbbcbac 100644 --- a/packages/cheerio-crawler/test/xml.test.ts +++ b/packages/cheerio-crawler/test/xml.test.ts @@ -3,7 +3,7 @@ import type { Server } from 'node:http'; import type { CheerioCrawlingContext } from '@crawlee/cheerio'; import { CheerioCrawler } from '@crawlee/cheerio'; -import { runExampleComServer } from '../../../test/shared/_helper'; +import { runExampleComServer } from '../../../test/shared/_helper.js'; let serverAddress = 'http://localhost:'; let port: number; diff --git a/packages/cli/package.json b/packages/cli/package.json index df3acb4985cc..39e45face15f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,22 +1,16 @@ { "name": "@crawlee/cli", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, "bin": { "crawlee": "./src/index.ts" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -51,12 +45,11 @@ "access": "public" }, "dependencies": { - "@crawlee/templates": "3.15.3", + "@crawlee/templates": "4.0.0", + "@inquirer/prompts": "^7.5.0", "ansi-colors": "^4.1.3", - "fs-extra": "^11.0.0", - "inquirer": "^8.2.4", - "tslib": "^2.4.0", - "yargonaut": "^1.1.4", - "yargs": "^17.5.1" + "fs-extra": "^11.3.0", + "tslib": "^2.8.1", + "yargs": "^18.0.0" } } diff --git a/packages/cli/src/commands/CreateProjectCommand.ts b/packages/cli/src/commands/CreateProjectCommand.ts index 20c25c301c2d..dd1f89d047f3 100644 --- a/packages/cli/src/commands/CreateProjectCommand.ts +++ b/packages/cli/src/commands/CreateProjectCommand.ts @@ -7,9 +7,9 @@ import { setTimeout } from 'node:timers/promises'; import type { Template } from '@crawlee/templates'; import { fetchManifest } from '@crawlee/templates'; +import { input, select } from '@inquirer/prompts'; import colors from 'ansi-colors'; -import { ensureDir } from 'fs-extra'; -import { prompt } from 'inquirer'; +import { ensureDir } from 'fs-extra/esm'; import type { ArgumentsCamelCase, Argv, CommandModule } from 'yargs'; interface CreateProjectArgs { @@ -138,22 +138,17 @@ export class CreateProjectCommand implements CommandModule { - try { - validateProjectName(promptText); - } catch (err: any) { - return err.message; - } - return true; - }, + projectName = await input({ + message: 'Name of the new project folder:', + validate: (promptText) => { + try { + validateProjectName(promptText); + } catch (err: any) { + return err.message; + } + return true; }, - ]); - ({ projectName } = projectNamePrompt); + }); } else { validateProjectName(projectName); } @@ -165,16 +160,11 @@ export class CreateProjectCommand implements CommandModule [options]') @@ -43,12 +36,14 @@ const cli = yargs .command(new RunProjectCommand()) .command(new InstallPlaywrightBrowsersCommand()) .recommendCommands() + .showHelpOnFail(true) + .demandCommand(1, '') .strict(); void (async () => { const args = (await cli.parse(process.argv.slice(2))) as { _: string[] }; if (args._.length === 0) { - yargs.showHelp(); + yargs(process.argv.slice(2)).showHelp(); } })(); diff --git a/packages/core/package.json b/packages/core/package.json index 176657ae99ea..8480d7a4185d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/core", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,34 +40,34 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/consts": "^2.20.0", - "@apify/datastructures": "^2.0.0", - "@apify/log": "^2.4.0", - "@apify/pseudo_url": "^2.0.30", - "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.7.10", - "@crawlee/memory-storage": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "@sapphire/async-queue": "^1.5.1", - "@vladfrangu/async_event_emitter": "^2.2.2", - "csv-stringify": "^6.2.0", - "fs-extra": "^11.0.0", - "got-scraping": "^4.0.0", + "@apify/consts": "^2.41.0", + "@apify/datastructures": "^2.0.3", + "@apify/log": "^2.5.18", + "@apify/pseudo_url": "^2.0.59", + "@apify/timeout": "^0.3.2", + "@apify/utilities": "^2.15.5", + "@crawlee/memory-storage": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "@sapphire/async-queue": "^1.5.5", + "@vladfrangu/async_event_emitter": "^2.4.6", + "csv-stringify": "^6.5.2", + "fs-extra": "^11.3.0", "json5": "^2.2.3", - "minimatch": "^9.0.0", - "ow": "^0.28.1", - "stream-json": "^1.8.0", - "tldts": "^7.0.0", + "minimatch": "^10.0.1", + "ow": "^2.0.0", + "stream-json": "^1.9.1", + "tldts": "^7.0.6", "tough-cookie": "^6.0.0", - "tslib": "^2.4.0", - "type-fest": "^4.0.0" + "tslib": "^2.8.1", + "type-fest": "^4.41.0", + "zod": "^4.0.0" } } diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.ts index 7bfa33f80707..bfc11c07b265 100644 --- a/packages/core/src/autoscaling/autoscaled_pool.ts +++ b/packages/core/src/autoscaling/autoscaled_pool.ts @@ -5,13 +5,13 @@ import { addTimeoutToPromise } from '@apify/timeout'; import type { BetterIntervalID } from '@apify/utilities'; import { betterClearInterval, betterSetInterval } from '@apify/utilities'; -import { Configuration } from '../configuration'; -import { CriticalError } from '../errors'; -import { log as defaultLog } from '../log'; -import type { SnapshotterOptions } from './snapshotter'; -import { Snapshotter } from './snapshotter'; -import type { SystemInfo, SystemStatusOptions } from './system_status'; -import { SystemStatus } from './system_status'; +import { Configuration } from '../configuration.js'; +import { CriticalError } from '../errors.js'; +import { log as defaultLog } from '../log.js'; +import type { SnapshotterOptions } from './snapshotter.js'; +import { Snapshotter } from './snapshotter.js'; +import type { SystemInfo, SystemStatusOptions } from './system_status.js'; +import { SystemStatus } from './system_status.js'; export interface AutoscaledPoolOptions { /** diff --git a/packages/core/src/autoscaling/index.ts b/packages/core/src/autoscaling/index.ts index 991e454b1988..328db1f1c3f3 100644 --- a/packages/core/src/autoscaling/index.ts +++ b/packages/core/src/autoscaling/index.ts @@ -1,3 +1,3 @@ -export * from './autoscaled_pool'; -export * from './snapshotter'; -export * from './system_status'; +export * from './autoscaled_pool.js'; +export * from './snapshotter.js'; +export * from './system_status.js'; diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.ts index 9792bc2c37e1..f6bcd801d2cc 100644 --- a/packages/core/src/autoscaling/snapshotter.ts +++ b/packages/core/src/autoscaling/snapshotter.ts @@ -1,16 +1,16 @@ import type { StorageClient } from '@crawlee/types'; -import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils'; +import { getMemoryInfo, isContainerized } from '@crawlee/utils'; import ow from 'ow'; import type { Log } from '@apify/log'; import type { BetterIntervalID } from '@apify/utilities'; import { betterClearInterval, betterSetInterval } from '@apify/utilities'; -import { Configuration } from '../configuration'; -import type { EventManager } from '../events/event_manager'; -import { EventType } from '../events/event_manager'; -import { log as defaultLog } from '../log'; -import type { SystemInfo } from './system_status'; +import { Configuration } from '../configuration.js'; +import type { EventManager } from '../events/event_manager.js'; +import { EventType } from '../events/event_manager.js'; +import { log as defaultLog } from '../log.js'; +import type { SystemInfo } from './system_status.js'; const RESERVE_MEMORY_RATIO = 0.5; const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2; @@ -195,16 +195,9 @@ export class Snapshotter { if (memoryMbytes > 0) { this.maxMemoryBytes = memoryMbytes * 1024 * 1024; } else { - let totalBytes: number; - - if (this.config.get('systemInfoV2')) { - const containerized = this.config.get('containerized', await isContainerized()); - const memInfo = await getMemoryInfoV2(containerized); - totalBytes = memInfo.totalBytes; - } else { - const memInfo = await getMemoryInfo(); - totalBytes = memInfo.totalBytes; - } + const containerized = this.config.get('containerized', await isContainerized()); + const memInfo = await getMemoryInfo(containerized); + const totalBytes = memInfo.totalBytes; this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio')!); this.log.debug( diff --git a/packages/core/src/autoscaling/system_status.ts b/packages/core/src/autoscaling/system_status.ts index b2b86434e323..862d8360dca6 100644 --- a/packages/core/src/autoscaling/system_status.ts +++ b/packages/core/src/autoscaling/system_status.ts @@ -1,8 +1,8 @@ import { weightedAvg } from '@crawlee/utils'; import ow from 'ow'; -import type { Configuration } from '../configuration'; -import { Snapshotter } from './snapshotter'; +import type { Configuration } from '../configuration.js'; +import { Snapshotter } from './snapshotter.js'; /** * Represents the current status of the system. diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.ts index 42b9c22db4b0..2807619c68bc 100644 --- a/packages/core/src/configuration.ts +++ b/packages/core/src/configuration.ts @@ -1,182 +1,192 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { EventEmitter } from 'node:events'; +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import type { MemoryStorageOptions } from '@crawlee/memory-storage'; import { MemoryStorage } from '@crawlee/memory-storage'; -import type { Dictionary, StorageClient } from '@crawlee/types'; -import { pathExistsSync, readFileSync } from 'fs-extra'; +import type { StorageClient } from '@crawlee/types'; +import { pathExistsSync } from 'fs-extra/esm'; +import { z } from 'zod'; import log, { LogLevel } from '@apify/log'; -import { type EventManager, LocalEventManager } from './events'; -import type { StorageManager } from './storages'; -import { type Constructor, entries } from './typedefs'; +import { type EventManager } from './events/event_manager.js'; +import { LocalEventManager } from './events/local_event_manager.js'; +import type { StorageManager } from './storages/storage_manager.js'; +import type { Constructor } from './typedefs.js'; -export interface ConfigurationOptions { - /** - * Defines storage client to be used. - * @default {@apilink MemoryStorage} - */ - storageClient?: StorageClient; - - /** - * Defines the Event Manager to be used. - * @default {@apilink EventManager} - */ - eventManager?: EventManager; - - /** - * Could be used to adjust the storage client behavior - * e.g. {@apilink MemoryStorageOptions} could be used to adjust the {@apilink MemoryStorage} behavior. - */ - storageClientOptions?: Dictionary; - - /** - * Default dataset id. - * - * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable. - * @default 'default' - */ - defaultDatasetId?: string; - - /** - * Defines whether to purge the default storage folders before starting the crawler run. - * - * Alternative to `CRAWLEE_PURGE_ON_START` environment variable. - * @default true - */ - purgeOnStart?: boolean; - - /** - * Default key-value store id. - * - * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable. - * @default 'default' - */ - defaultKeyValueStoreId?: string; - - /** - * Default request queue id. - * - * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable. - * @default 'default' - */ - defaultRequestQueueId?: string; - - /** - * Sets the ratio, defining the maximum CPU usage. - * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded. - * @default 0.95 - */ - maxUsedCpuRatio?: number; - - /** - * Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}. - * When the memory usage is more than the provided ratio, the memory is considered overloaded. - * - * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable. - * @default 0.25 - */ - availableMemoryRatio?: number; - - /** - * Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}. - * By default, the maximum memory is set to one quarter of total system memory. - * - * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable. - */ - memoryMbytes?: number; +// ============================================================================ +// Field Definition Helpers +// ============================================================================ - /** - * Defines the interval of emitting the `persistState` event. - * - * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable. - * @default 60_000 - */ - persistStateIntervalMillis?: number; - - /** - Defines the interval of emitting the `systemInfo` event. - @default 1_000 - */ - systemInfoIntervalMillis?: number; - - /** - * Defines the default input key, i.e. the key that is used to get the crawler input value - * from the default {@apilink KeyValueStore} associated with the current crawler run. - * - * Alternative to `CRAWLEE_INPUT_KEY` environment variable. - * @default 'INPUT' - */ - inputKey?: string; - - /** - * Defines whether web browsers launched by Crawlee will run in the headless mode. - * - * Alternative to `CRAWLEE_HEADLESS` environment variable. - * @default true - */ - headless?: boolean; - - /** - * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee. - * - * Alternative to `CRAWLEE_XVFB` environment variable. - * @default false - */ - xvfb?: boolean; - - /** - * Defines a path to Chrome executable. - * - * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable. - */ - chromeExecutablePath?: string; +/** + * Defines a configuration field with its schema and optional environment variable mapping. + */ +export function field(schema: T, options: { env?: string | string[] } = {}): ConfigField { + const envKeys = options.env ? (Array.isArray(options.env) ? options.env : [options.env]) : []; + return { schema, envKeys }; +} - /** - * Defines a path to default browser executable. - * - * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable. - */ - defaultBrowserPath?: string; +export interface ConfigField { + schema: T; + envKeys: string[]; +} - /** - * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`. - * - * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable. - */ - disableBrowserSandbox?: boolean; +export type FieldDefinitions = Record; - /** - * Sets the log level to the given value. - * - * Alternative to `CRAWLEE_LOG_LEVEL` environment variable. - * @default 'INFO' - */ - logLevel?: LogLevel | LogLevel[keyof LogLevel]; +/** + * Infer the input options type from field definitions. + * All fields are optional for constructor input since they have defaults or env var fallbacks. + */ +export type InferInputOptions = { + [K in keyof T]?: z.input; +}; - /** - * Defines whether the storage client used should persist the data it stores. - * - * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable. - */ - persistStorage?: boolean; +/** + * Infer the output options type from field definitions. + * Respects Zod's output types, so fields with defaults are non-optional. + */ +export type InferOutputOptions = { + [K in keyof T]: z.output; +}; + +// ============================================================================ +// Zod Schemas for Complex Types +// ============================================================================ + +const storageClientSchema = z.custom((val) => val != null); +const eventManagerSchema = z.custom((val) => val != null); +const dictionarySchema = z.record(z.unknown()); + +/** Boolean coercion that treats '0', 'false', '' as falsy */ +export const coerceBoolean = z.preprocess((val) => { + if (typeof val === 'string') { + return !['0', 'false', ''].includes(val.toLowerCase()); + } + return Boolean(val); +}, z.boolean()); + +/** Log level schema that accepts both string names and numeric values */ +export const logLevelSchema = z.preprocess((val) => { + if (val == null) return undefined; + if (typeof val === 'number') return val; + if (typeof val === 'string') { + const num = Number(val); + if (Number.isFinite(num)) return num; + return LogLevel[val.toUpperCase() as keyof typeof LogLevel]; + } + return val; +}, z.nativeEnum(LogLevel).optional()); - /** - * Defines whether to use the systemInfoV2 metric collection experiment. - * - * Alternative to `CRAWLEE_SYSTEM_INFO_V2` environment variable. - */ - systemInfoV2?: boolean; +// ============================================================================ +// Crawlee Configuration Field Definitions +// ============================================================================ - /** - * Used in place of `isContainerized()` when collecting system metrics. - * - * Alternative to `CRAWLEE_CONTAINERIZED` environment variable. - */ - containerized?: boolean; -} +/** + * Field definitions for Crawlee Configuration. + * Each field defines its Zod schema and optional environment variable mapping. + * + * To extend in Apify SDK: + * ```ts + * const apifyConfigFields = { + * ...crawleeConfigFields, + * token: field(z.string().optional(), { env: 'APIFY_TOKEN' }), + * actorId: field(z.string().optional(), { env: ['ACTOR_ID', 'APIFY_ACTOR_ID'] }), + * }; + * ``` + */ +export const crawleeConfigFields = { + // Storage clients (no env vars, constructor only) + storageClient: field(storageClientSchema.optional()), + eventManager: field(eventManagerSchema.optional()), + storageClientOptions: field(dictionarySchema.default({})), + + // Storage IDs + defaultDatasetId: field(z.string().default('default'), { + env: 'CRAWLEE_DEFAULT_DATASET_ID', + }), + defaultKeyValueStoreId: field(z.string().default('default'), { + env: 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID', + }), + defaultRequestQueueId: field(z.string().default('default'), { + env: 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID', + }), + + // Storage behavior + purgeOnStart: field(coerceBoolean.default(true), { + env: 'CRAWLEE_PURGE_ON_START', + }), + persistStorage: field(coerceBoolean.default(true), { + env: 'CRAWLEE_PERSIST_STORAGE', + }), + + // Memory and CPU limits + maxUsedCpuRatio: field(z.coerce.number().default(0.95)), + availableMemoryRatio: field(z.coerce.number().default(0.25), { + env: 'CRAWLEE_AVAILABLE_MEMORY_RATIO', + }), + memoryMbytes: field(z.coerce.number().optional(), { + env: 'CRAWLEE_MEMORY_MBYTES', + }), + + // Intervals + persistStateIntervalMillis: field(z.coerce.number().default(60_000), { + env: 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', + }), + systemInfoIntervalMillis: field(z.coerce.number().default(1_000)), + + // Input + inputKey: field(z.string().default('INPUT'), { + env: 'CRAWLEE_INPUT_KEY', + }), + + // Browser options + headless: field(coerceBoolean.default(true), { + env: 'CRAWLEE_HEADLESS', + }), + xvfb: field(coerceBoolean.default(false), { + env: 'CRAWLEE_XVFB', + }), + chromeExecutablePath: field(z.string().optional(), { + env: 'CRAWLEE_CHROME_EXECUTABLE_PATH', + }), + defaultBrowserPath: field(z.string().optional(), { + env: 'CRAWLEE_DEFAULT_BROWSER_PATH', + }), + disableBrowserSandbox: field(coerceBoolean.optional(), { + env: 'CRAWLEE_DISABLE_BROWSER_SANDBOX', + }), + + // Logging + logLevel: field(logLevelSchema, { + env: 'CRAWLEE_LOG_LEVEL', + }), + + // System info + systemInfoV2: field(coerceBoolean.default(true), { + env: 'CRAWLEE_SYSTEM_INFO_V2', + }), + containerized: field(coerceBoolean.optional(), { + env: 'CRAWLEE_CONTAINERIZED', + }), +} as const; + +export type CrawleeConfigFields = typeof crawleeConfigFields; + +// ============================================================================ +// Configuration Options Types +// ============================================================================ + +/** Input options for Configuration constructor (all fields optional) */ +export type ConfigurationOptions = InferInputOptions; + +/** Output options from Configuration.get() (respects defaults) */ +export type ConfigurationValues = InferOutputOptions; + +// ============================================================================ +// Configuration Class +// ============================================================================ /** * `Configuration` is a value object holding Crawlee configuration. By default, there is a @@ -209,12 +219,12 @@ export interface ConfigurationOptions { * const crawler = new BasicCrawler({ ... }, config); * ``` * - * The configuration provided via environment variables always takes precedence. We can also - * define the `crawlee.json` file in the project root directory which will serve as a baseline, - * so the options provided in constructor will override those. In other words, the precedence is: + * The configuration provided via constructor always takes precedence. Environment variables + * come second, followed by `crawlee.json` file in the project root directory. In other words, + * the precedence is: * * ```text - * crawlee.json < constructor options < environment variables + * constructor options > environment variables > crawlee.json > defaults * ``` * * ## Supported Configuration Options @@ -242,61 +252,46 @@ export interface ConfigurationOptions { * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | - * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25` * `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false - * `containerized | `CRAWLEE_CONTAINERIZED | - + * `containerized` | `CRAWLEE_CONTAINERIZED` | - */ -export class Configuration { +export class Configuration< + TFields extends FieldDefinitions = CrawleeConfigFields, + TInput extends InferInputOptions = InferInputOptions, + TOutput extends InferOutputOptions = InferOutputOptions, +> { /** - * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`) + * Field definitions for this configuration class. + * Override in subclasses to add new fields. */ - protected static ENV_MAP: Dictionary = { - CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio', - CRAWLEE_PURGE_ON_START: 'purgeOnStart', - CRAWLEE_MEMORY_MBYTES: 'memoryMbytes', - CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId', - CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId', - CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId', - CRAWLEE_INPUT_KEY: 'inputKey', - CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis', - CRAWLEE_HEADLESS: 'headless', - CRAWLEE_XVFB: 'xvfb', - CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath', - CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath', - CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox', - CRAWLEE_LOG_LEVEL: 'logLevel', - CRAWLEE_PERSIST_STORAGE: 'persistStorage', - CRAWLEE_SYSTEM_INFO_V2: 'systemInfoV2', - CRAWLEE_CONTAINERIZED: 'containerized', - }; - - protected static BOOLEAN_VARS = [ - 'purgeOnStart', - 'headless', - 'xvfb', - 'disableBrowserSandbox', - 'persistStorage', - 'systemInfoV2', - 'containerized', - ]; - - protected static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis']; - - protected static COMMA_SEPARATED_LIST_VARS: string[] = []; - - protected static DEFAULTS: Dictionary = { - defaultKeyValueStoreId: 'default', - defaultDatasetId: 'default', - defaultRequestQueueId: 'default', - inputKey: 'INPUT', - maxUsedCpuRatio: 0.95, - availableMemoryRatio: 0.25, - storageClientOptions: {}, - purgeOnStart: true, - headless: true, - persistStateIntervalMillis: 60_000, - systemInfoIntervalMillis: 1_000, - persistStorage: true, - systemInfoV2: true, - }; + static fields: FieldDefinitions = crawleeConfigFields; + + /** + * Extends an existing field with additional environment variable mappings. + * The new env vars are checked first, then the base field's env vars. + * Intended for use when extending Configuration in other packages (e.g., Apify SDK). + * + * @internal + * @example + * ```ts + * const apifyConfigFields = { + * ...crawleeConfigFields, + * // Adds ACTOR_* and APIFY_* aliases, keeps CRAWLEE_* from base + * defaultDatasetId: Configuration.extendField(crawleeConfigFields.defaultDatasetId, { + * env: ['ACTOR_DEFAULT_DATASET_ID', 'APIFY_DEFAULT_DATASET_ID'], + * }), + * }; + * ``` + */ + static extendField( + baseField: ConfigField, + options: { env?: string | string[] } = {}, + ): ConfigField { + const newEnvKeys = options.env ? (Array.isArray(options.env) ? options.env : [options.env]) : []; + return { + schema: baseField.schema, + envKeys: [...newEnvKeys, ...baseField.envKeys], + }; + } /** * Provides access to the current-instance-scoped Configuration without passing it around in parameters. @@ -304,133 +299,136 @@ export class Configuration { */ static storage = new AsyncLocalStorage(); - protected options!: Map; - protected services = new Map(); - /** @internal */ static globalConfig?: Configuration; + protected options = new Map(); + protected services = new Map(); + protected userOptions = new Set(); + public readonly storageManagers = new Map(); /** - * Creates new `Configuration` instance with provided options. Env vars will have precedence over those. + * Creates new `Configuration` instance with provided options. + * Constructor options take precedence over environment variables. */ - constructor(options: ConfigurationOptions = {}) { + constructor(options: TInput = {} as TInput) { this.buildOptions(options); // Increase the global limit for event emitter memory leak warnings. EventEmitter.defaultMaxListeners = 50; - // set the log level to support CRAWLEE_ prefixed env var too - const logLevel = this.get('logLevel'); + // Set the log level + const logLevel = this.get('logLevel' as keyof TOutput); - if (logLevel) { - const level = Number.isFinite(+logLevel) - ? +logLevel - : LogLevel[String(logLevel).toUpperCase() as unknown as LogLevel]; - log.setLevel(level as LogLevel); + if (logLevel != null) { + log.setLevel(logLevel as LogLevel); } } /** - * Returns configured value. First checks the environment variables, then provided configuration, - * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described - * in the above section. + * Returns the field definitions for this configuration class. + * Uses the static `fields` property from the actual class (supports inheritance). */ - get(key: T, defaultValue?: U): U { - // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option - let envValue: string | undefined; - - for (const [k, v] of entries(Configuration.ENV_MAP)) { - if (key === v) { - envValue = process.env[k as string]; + protected getFields(): TFields { + return (this.constructor as typeof Configuration).fields as TFields; + } - if (envValue) { - break; - } - } - } + /** + * Returns configured value. First checks constructor options, then environment variables, + * then crawlee.json values, and finally falls back to the default value. + */ + get(key: K, defaultValue: NonNullable): NonNullable; + get(key: K, defaultValue?: TOutput[K]): TOutput[K]; + get(key: K, defaultValue?: TOutput[K]): TOutput[K] { + const fields = this.getFields(); + const fieldDef = fields[key as string] as ConfigField | undefined; - if (envValue != null) { - return this._castEnvValue(key, envValue) as U; + // 1. Constructor options take precedence + if (this.userOptions.has(key as keyof TInput) && this.options.has(key as keyof TInput)) { + return this.options.get(key as keyof TInput) as TOutput[K]; } - // check instance level options - if (this.options.has(key)) { - return this.options.get(key) as U; + // 2. Check environment variables + if (fieldDef?.envKeys.length) { + for (const envKey of fieldDef.envKeys) { + const envValue = process.env[envKey]; + if (envValue != null && envValue !== '') { + // Parse through the field's schema for type coercion + const parsed = fieldDef.schema.safeParse(envValue); + if (parsed.success) { + return parsed.data as TOutput[K]; + } + } + } } - // fallback to defaults - return (defaultValue ?? Configuration.DEFAULTS[key as keyof typeof Configuration.DEFAULTS] ?? envValue) as U; - } - - protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean) { - if (Configuration.INTEGER_VARS.includes(key)) { - return +value; + // 3. Check options from crawlee.json (stored in options but not in userOptions) + if (this.options.has(key as keyof TInput)) { + return this.options.get(key as keyof TInput) as TOutput[K]; } - if (Configuration.BOOLEAN_VARS.includes(key)) { - // 0, false and empty string are considered falsy values - return !['0', 'false', ''].includes(String(value).toLowerCase()); + // 4. Fall back to schema default or provided default + if (defaultValue !== undefined) { + return defaultValue; } - if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) { - if (!value) return []; - return String(value) - .split(',') - .map((v) => v.trim()); + if (fieldDef) { + const parsed = fieldDef.schema.safeParse(undefined); + if (parsed.success) { + return parsed.data as TOutput[K]; + } } - return value; + return undefined as TOutput[K]; } /** - * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var. + * Sets value for given option. Only affects this `Configuration` instance, + * the value will not be propagated down to the env var. * To reset a value, we can omit the `value` argument or pass `undefined` there. */ - set(key: keyof ConfigurationOptions, value?: any): void { + set(key: K, value?: TInput[K]): void { this.options.set(key, value); + this.userOptions.add(key); } /** - * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var. - * To reset a value, we can omit the `value` argument or pass `undefined` there. + * Sets value for given option on the global configuration instance. */ - static set(key: keyof ConfigurationOptions, value?: any): void { + static set(key: K, value?: ConfigurationOptions[K]): void { this.getGlobalConfig().set(key, value); } /** * Returns cached instance of {@apilink StorageClient} using options as defined in the environment variables or in - * this {@apilink Configuration} instance. Only first call of this method will create the client, following calls will - * return the same client instance. - * - * Caching works based on the `storageClientOptions`, so calling this method with different options will return - * multiple instances, one for each variant of the options. + * this {@apilink Configuration} instance. * @internal */ getStorageClient(): StorageClient { - if (this.options.has('storageClient')) { - return this.options.get('storageClient') as StorageClient; + const storageClient = this.options.get('storageClient' as keyof TInput); + if (storageClient) { + return storageClient as StorageClient; } - const options = this.options.get('storageClientOptions') as Dictionary; + const options = this.get('storageClientOptions' as keyof TOutput) as Record | undefined; return this.createMemoryStorage(options); } getEventManager(): EventManager { - if (this.options.has('eventManager')) { - return this.options.get('eventManager') as EventManager; + const eventManager = this.options.get('eventManager' as keyof TInput); + if (eventManager) { + return eventManager as EventManager; } if (this.services.has('eventManager')) { return this.services.get('eventManager') as EventManager; } - const eventManager = new LocalEventManager(this); - this.services.set('eventManager', eventManager); + const newEventManager = new LocalEventManager(this as unknown as Configuration); + this.services.set('eventManager', newEventManager); - return eventManager; + return newEventManager; } /** @@ -445,8 +443,7 @@ export class Configuration { } const storage = new MemoryStorage({ - persistStorage: this.get('persistStorage'), - // Override persistStorage if user provides it via storageClientOptions + persistStorage: this.get('persistStorage' as keyof TOutput) as boolean | undefined, ...options, }); this.services.set(cacheKey, storage); @@ -455,7 +452,8 @@ export class Configuration { } useStorageClient(client: StorageClient): void { - this.options.set('storageClient', client); + this.options.set('storageClient' as keyof TInput, client as TInput[keyof TInput]); + this.userOptions.add('storageClient' as keyof TInput); } static useStorageClient(client: StorageClient): void { @@ -463,7 +461,8 @@ export class Configuration { } useEventManager(events: EventManager): void { - this.options.set('eventManager', events); + this.options.set('eventManager' as keyof TInput, events as TInput[keyof TInput]); + this.userOptions.add('eventManager' as keyof TInput); } /** @@ -500,20 +499,27 @@ export class Configuration { delete this.globalConfig; } - protected buildOptions(options: ConfigurationOptions) { - // try to load configuration from crawlee.json as the baseline + protected buildOptions(options: TInput) { + // Track which options were explicitly provided by the user + this.userOptions = new Set(Object.keys(options) as (keyof TInput)[]); + + // Try to load configuration from crawlee.json as the baseline const path = join(process.cwd(), 'crawlee.json'); if (pathExistsSync(path)) { try { const file = readFileSync(path); const optionsFromFileConfig = JSON.parse(file.toString()); - Object.assign(options, optionsFromFileConfig); + // File config is baseline, user options override + options = { ...optionsFromFileConfig, ...options }; } catch { // ignore } } - this.options = new Map(entries(options)); + // Store all options + for (const [key, value] of Object.entries(options)) { + this.options.set(key as keyof TInput, value); + } } } diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.ts index a97477ce7370..6598ec3f30c3 100644 --- a/packages/core/src/cookie_utils.ts +++ b/packages/core/src/cookie_utils.ts @@ -1,8 +1,8 @@ import type { Cookie as CookieObject } from '@crawlee/types'; import { Cookie, CookieJar } from 'tough-cookie'; -import { log } from './log'; -import { CookieParseError } from './session_pool/errors'; +import { log } from './log.js'; +import { CookieParseError } from './session_pool/errors.js'; export interface ResponseLike { url?: string | (() => string); @@ -12,16 +12,14 @@ export interface ResponseLike { /** * @internal */ -export function getCookiesFromResponse(response: ResponseLike): Cookie[] { - const headers = typeof response.headers === 'function' ? response.headers() : response.headers; - const cookieHeader = headers?.['set-cookie'] || ''; +export function getCookiesFromResponse(response: Response): Cookie[] { + const headers = response.headers; + const cookieHeaders = headers.getSetCookie(); try { - return Array.isArray(cookieHeader) - ? cookieHeader.map((cookie) => Cookie.parse(cookie)!) - : [Cookie.parse(cookieHeader)!]; + return cookieHeaders.map((cookie) => Cookie.parse(cookie)!); } catch (e) { - throw new CookieParseError(cookieHeader); + throw new CookieParseError(cookieHeaders); } } @@ -122,7 +120,7 @@ export function mergeCookies(url: string, sourceCookies: string[]): string { }); if (similarKeyCookie) { - log.deprecated( + log.warningOnce( `Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`, ); } diff --git a/packages/core/src/crawlers/context_pipeline.ts b/packages/core/src/crawlers/context_pipeline.ts new file mode 100644 index 000000000000..e84361a251eb --- /dev/null +++ b/packages/core/src/crawlers/context_pipeline.ts @@ -0,0 +1,170 @@ +import type { Awaitable } from '@crawlee/types'; + +import { + ContextPipelineCleanupError, + ContextPipelineInitializationError, + ContextPipelineInterruptedError, + RequestHandlerError, + SessionError, +} from '../errors.js'; + +/** + * Represents a middleware step in the context pipeline. + * + * @template TCrawlingContext - The input context type for this middleware + * @template TCrawlingContextExtension - The enhanced output context type + */ +export interface ContextMiddleware { + /** The main middleware function that enhances the context */ + action: (context: TCrawlingContext) => Awaitable; + /** Optional cleanup function called after the consumer finishes or fails */ + cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable; +} + +/** + * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities. + * + * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation. + * This class provides a type-safe way to build a pipeline of context transformations where each step + * can enhance the context with additional properties or utilities. + * + * @template TContextBase - The base context type that serves as the starting point + * @template TCrawlingContext - The final context type after all middleware transformations + */ +export abstract class ContextPipeline { + /** + * Creates a new empty context pipeline. + * + * @template TContextBase - The base context type for the pipeline + * @returns A new ContextPipeline instance with no transformations + */ + static create(): ContextPipeline { + return new ContextPipelineImpl({ action: async (context) => context }); + } + + /** + * Adds a middleware to the pipeline, creating a new pipeline instance. + * + * This method provides a fluent interface for building context transformation pipelines. + * Each middleware can enhance the context with additional properties or utilities. + * + * @template TCrawlingContextExtension - The enhanced context type produced by this middleware + * @param middleware - The middleware to add to the pipeline + * @returns A new ContextPipeline instance with the added middleware + */ + abstract compose( + middleware: ContextMiddleware, + ): ContextPipeline; + + /** + * Executes the middleware pipeline and passes the final context to a consumer function. + * + * This method runs the crawling context through the entire middleware chain, enhancing it + * at each step, and then passes the final enhanced context to the provided consumer function. + * Proper cleanup is performed even if exceptions occur during processing. + * + * @param crawlingContext - The initial context to process through the pipeline + * @param finalContextConsumer - The function that will receive the final enhanced context + * + * @throws {ContextPipelineInitializationError} When a middleware fails during initialization + * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization + * @throws {RequestHandlerError} When the final context consumer throws an exception + * @throws {ContextPipelineCleanupError} When cleanup operations fail + * @throws {SessionError} Session errors are re-thrown as-is for special handling + */ + abstract call( + crawlingContext: TContextBase, + finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable, + ): Promise; +} + +/** + * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent` + * properties from the `ContextPipeline` interface, making type checking more reliable. + */ +class ContextPipelineImpl extends ContextPipeline< + TContextBase, + TCrawlingContext +> { + constructor( + private middleware: ContextMiddleware, + private parent?: ContextPipelineImpl, + ) { + super(); + } + + /** + * @inheritdoc + */ + compose( + middleware: ContextMiddleware, + ): ContextPipeline { + return new ContextPipelineImpl( + middleware as any, + this as any, + ); + } + + private *middlewareChain() { + let step: ContextPipelineImpl | undefined = this as any; + + while (step !== undefined) { + yield step.middleware; + step = step.parent; + } + } + + /** + * @inheritdoc + */ + async call( + crawlingContext: TContextBase, + finalContextConsumer: (finalContext: TCrawlingContext) => Promise, + ): Promise { + const middlewares = Array.from(this.middlewareChain()).reverse(); + const cleanupStack = []; + let consumerException: unknown | undefined; + + try { + for (const { action, cleanup } of middlewares) { + try { + const contextExtension = await action(crawlingContext); + Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension)); + + if (cleanup) { + cleanupStack.push(cleanup); + } + } catch (exception: unknown) { + if (exception instanceof SessionError) { + throw exception; // Session errors are re-thrown as-is + } + if (exception instanceof ContextPipelineInterruptedError) { + throw exception; + } + + throw new ContextPipelineInitializationError(exception); + } + } + + try { + await finalContextConsumer(crawlingContext as TCrawlingContext); + } catch (exception: unknown) { + if (exception instanceof SessionError) { + consumerException = exception; + throw exception; // Session errors are re-thrown as-is + } + consumerException = exception; + throw new RequestHandlerError(exception); + } + } finally { + try { + for (const cleanup of cleanupStack.reverse()) { + await cleanup(crawlingContext, consumerException); + } + } catch (exception: unknown) { + // eslint-disable-next-line no-unsafe-finally + throw new ContextPipelineCleanupError(exception); + } + } + } +} diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts index 773bca2c3031..4a11429b20ad 100644 --- a/packages/core/src/crawlers/crawler_commons.ts +++ b/packages/core/src/crawlers/crawler_commons.ts @@ -1,16 +1,14 @@ -import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types'; -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { OptionsInit, Response as GotResponse } from 'got-scraping'; -import type { ReadonlyDeep } from 'type-fest'; - -import type { Configuration } from '../configuration'; -import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links'; -import type { Log } from '../log'; -import type { ProxyInfo } from '../proxy_configuration'; -import type { Request, Source } from '../request'; -import type { Session } from '../session_pool/session'; -import type { Dataset, RecordOptions, RequestQueueOperationOptions } from '../storages'; -import { KeyValueStore } from '../storages'; +import type { Dictionary, HttpRequestOptions, ProxyInfo, SendRequestOptions } from '@crawlee/types'; +import type { ReadonlyDeep, SetRequired } from 'type-fest'; + +import type { Configuration } from '../configuration.js'; +import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js'; +import type { Log } from '../log.js'; +import type { Request, Source } from '../request.js'; +import type { Session } from '../session_pool/session.js'; +import type { Dataset } from '../storages/dataset.js'; +import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js'; +import type { RequestQueueOperationOptions } from '../storages/request_provider.js'; /** @internal */ export type IsAny = 0 extends 1 & T ? true : false; @@ -21,15 +19,14 @@ export type WithRequired = T & { [P in K]-?: T[P] }; export type LoadedRequest = WithRequired; /** @internal */ -export type LoadedContext = IsAny extends true - ? Context - : { - request: LoadedRequest; - } & Omit; - -export interface RestrictedCrawlingContext - // we need `Record` here, otherwise `Omit` is resolved badly - extends Record { +export type LoadedContext = + IsAny extends true + ? Context + : { + request: LoadedRequest; + } & Omit; + +export interface RestrictedCrawlingContext { id: string; session?: Session; @@ -77,7 +74,9 @@ export interface RestrictedCrawlingContext>) => Promise; + enqueueLinks: ( + options: ReadonlyDeep, 'requestQueue' | 'robotsTxtFile'>>, + ) => Promise; /** * Add requests directly to the request queue. @@ -108,10 +107,7 @@ export interface RestrictedCrawlingContext - extends RestrictedCrawlingContext { - crawler: Crawler; - +export interface CrawlingContext extends RestrictedCrawlingContext { /** * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue} * currently used by the crawler. @@ -138,17 +134,12 @@ export interface CrawlingContext> & Pick, - ): Promise; - - /** - * Get a key-value store with given name or id, or the default one for the crawler. - */ - getKeyValueStore: (idOrName?: string) => Promise; + options: ReadonlyDeep, 'requestQueue' | 'robotsTxtFile'>> & + Pick, + ): Promise; /** - * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request - * options on the fly. + * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly. * * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests). * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for @@ -163,7 +154,15 @@ export interface CrawlingContext(overrideOptions?: Partial): Promise>; + sendRequest: ( + requestOverrides?: Partial, + optionsOverrides?: SendRequestOptions, + ) => Promise; + + /** + * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance. + */ + registerDeferredCleanup(cleanup: () => Promise): void; } /** diff --git a/packages/core/src/crawlers/crawler_extension.ts b/packages/core/src/crawlers/crawler_extension.ts deleted file mode 100644 index c098d6c15a61..000000000000 --- a/packages/core/src/crawlers/crawler_extension.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { type Log, log as defaultLog } from '../log'; - -/** - * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method. - * @category Crawlers - * @ignore - */ -export abstract class CrawlerExtension { - name = this.constructor.name; - log: Log = defaultLog.child({ prefix: this.name }); - - getCrawlerOptions(): Record { - throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`); - } -} diff --git a/packages/core/src/crawlers/crawler_utils.ts b/packages/core/src/crawlers/crawler_utils.ts index 058132afaa3d..77752c0450be 100644 --- a/packages/core/src/crawlers/crawler_utils.ts +++ b/packages/core/src/crawlers/crawler_utils.ts @@ -1,6 +1,6 @@ import { TimeoutError } from '@apify/timeout'; -import type { Session } from '../session_pool/session'; +import type { Session } from '../session_pool/session.js'; /** * Handles timeout request diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.ts index 96af2f3f49e3..441597cc8475 100644 --- a/packages/core/src/crawlers/error_snapshotter.ts +++ b/packages/core/src/crawlers/error_snapshotter.ts @@ -1,18 +1,15 @@ import crypto from 'node:crypto'; -import type { CrawlingContext } from '../crawlers/crawler_commons'; -import type { KeyValueStore } from '../storages'; -import type { ErrnoException } from './error_tracker'; +import type { CrawlingContext } from '../crawlers/crawler_commons.js'; +import type { KeyValueStore } from '../storages/key_value_store.js'; +import type { ErrnoException } from './error_tracker.js'; +import type { SnapshottableProperties } from './internals/types.js'; // Define the following types as we cannot import the complete types from the respective packages interface BrowserCrawlingContext { saveSnapshot: (options: { key: string }) => Promise; } -interface BrowserPage { - content: () => Promise; -} - export interface SnapshotResult { screenshotFileName?: string; htmlFileName?: string; @@ -49,9 +46,12 @@ export class ErrorSnapshotter { /** * Capture a snapshot of the error context. */ - async captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise { + async captureSnapshot( + error: ErrnoException, + context: CrawlingContext & SnapshottableProperties, + ): Promise { try { - const page = context?.page as BrowserPage | undefined; + const page = context?.page; const body = context?.body; const keyValueStore = await context?.getKeyValueStore(); @@ -88,9 +88,9 @@ export class ErrorSnapshotter { return { screenshotFileName, - screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName), + screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)), htmlFileName, - htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName), + htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)), }; } catch { return {}; @@ -120,7 +120,11 @@ export class ErrorSnapshotter { /** * Save the HTML snapshot of the page, and return the fileName with the extension. */ - async saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise { + async saveHTMLSnapshot( + html: string, + keyValueStore: Pick, + fileName: string, + ): Promise { try { await keyValueStore.setValue(fileName, html, { contentType: 'text/html' }); return `${fileName}.html`; diff --git a/packages/core/src/crawlers/error_tracker.ts b/packages/core/src/crawlers/error_tracker.ts index eefa2f2c914f..fa085a188a64 100644 --- a/packages/core/src/crawlers/error_tracker.ts +++ b/packages/core/src/crawlers/error_tracker.ts @@ -1,7 +1,8 @@ import { inspect } from 'node:util'; -import type { CrawlingContext } from '../crawlers/crawler_commons'; -import { ErrorSnapshotter } from './error_snapshotter'; +import type { CrawlingContext } from '../crawlers/crawler_commons.js'; +import { ErrorSnapshotter } from './error_snapshotter.js'; +import type { SnapshottableProperties } from './internals/types.js'; /** * Node.js Error interface @@ -405,7 +406,11 @@ export class ErrorTracker { return result.sort((a, b) => b[0] - a[0]).slice(0, count); } - async captureSnapshot(storage: Record, error: ErrnoException, context: CrawlingContext) { + async captureSnapshot( + storage: Record, + error: ErrnoException, + context: CrawlingContext & SnapshottableProperties, + ) { if (!this.errorSnapshotter) { return; } diff --git a/packages/core/src/crawlers/index.ts b/packages/core/src/crawlers/index.ts index 77a83511e413..5fa44b458a4c 100644 --- a/packages/core/src/crawlers/index.ts +++ b/packages/core/src/crawlers/index.ts @@ -1,6 +1,6 @@ -export * from './crawler_commons'; -export * from './crawler_extension'; -export * from './crawler_utils'; -export * from './statistics'; -export * from './error_tracker'; -export * from './error_snapshotter'; +export * from './context_pipeline.js'; +export * from './crawler_commons.js'; +export * from './crawler_utils.js'; +export * from './statistics.js'; +export * from './error_tracker.js'; +export * from './error_snapshotter.js'; diff --git a/packages/core/src/crawlers/internals/types.ts b/packages/core/src/crawlers/internals/types.ts new file mode 100644 index 000000000000..f631f17acbc0 --- /dev/null +++ b/packages/core/src/crawlers/internals/types.ts @@ -0,0 +1,8 @@ +export interface BrowserPage { + content: () => Promise; +} + +export interface SnapshottableProperties { + body?: unknown; + page?: BrowserPage; +} diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts index 975a6537984f..485d6738a9fc 100644 --- a/packages/core/src/crawlers/statistics.ts +++ b/packages/core/src/crawlers/statistics.ts @@ -2,12 +2,12 @@ import ow from 'ow'; import type { Log } from '@apify/log'; -import { Configuration } from '../configuration'; -import type { EventManager } from '../events/event_manager'; -import { EventType } from '../events/event_manager'; -import { log as defaultLog } from '../log'; -import { KeyValueStore } from '../storages/key_value_store'; -import { ErrorTracker } from './error_tracker'; +import { Configuration } from '../configuration.js'; +import type { EventManager } from '../events/event_manager.js'; +import { EventType } from '../events/event_manager.js'; +import { log as defaultLog } from '../log.js'; +import { KeyValueStore } from '../storages/key_value_store.js'; +import { ErrorTracker } from './error_tracker.js'; /** * @ignore @@ -72,7 +72,7 @@ export class Statistics { /** * Statistic instance id. */ - readonly id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV + readonly id: string; /** * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls @@ -90,7 +90,7 @@ export class Statistics { private readonly config: Configuration; protected keyValueStore?: KeyValueStore = undefined; - protected persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`; + protected persistStateKey: string; private logIntervalMillis: number; private logMessage: string; private listener: () => Promise; @@ -115,6 +115,7 @@ export class Statistics { config: ow.optional.object, persistenceOptions: ow.optional.object, saveErrorSnapshots: ow.optional.boolean, + id: ow.optional.any(ow.number, ow.string), }), ); @@ -127,8 +128,12 @@ export class Statistics { enable: true, }, saveErrorSnapshots = false, + id, } = options; + this.id = id ?? String(Statistics.id++); + this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`; + this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' }); this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots }); this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots }); @@ -474,6 +479,16 @@ export interface StatisticsOptions { * @default false */ saveErrorSnapshots?: boolean; + + /** + * A unique identifier for this statistics instance. This ID is used for persistence + * to the key value store, ensuring the same statistics can be loaded after script restarts. + * + * If not provided, an auto-incremented ID will be used for backward compatibility. + * This means statistics may not persist correctly across script restarts + * if crawler creation order changes. + */ + id?: string; } /** @@ -481,7 +496,7 @@ export interface StatisticsOptions { */ export interface StatisticPersistedState extends Omit { requestRetryHistogram: number[]; - statsId: number; + statsId: string; requestAvgFailedDurationMillis: number; requestAvgFinishedDurationMillis: number; requestTotalDurationMillis: number; diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts index 5d6d2fce0e55..cc6cee467cc6 100644 --- a/packages/core/src/enqueue_links/enqueue_links.ts +++ b/packages/core/src/enqueue_links/enqueue_links.ts @@ -6,13 +6,13 @@ import type { SetRequired } from 'type-fest'; import log from '@apify/log'; -import type { Request, RequestOptions } from '../request'; +import type { Request, RequestOptions } from '../request.js'; import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions, -} from '../storages'; +} from '../storages/request_provider.js'; import type { GlobInput, PseudoUrlInput, @@ -21,7 +21,7 @@ import type { SkippedRequestCallback, SkippedRequestReason, UrlPatternObject, -} from './shared'; +} from './shared.js'; import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, @@ -29,7 +29,7 @@ import { createRequestOptions, createRequests, filterRequestsByPatterns, -} from './shared'; +} from './shared.js'; export interface EnqueueLinksOptions extends RequestQueueOperationOptions { /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */ @@ -291,7 +291,7 @@ export async function enqueueLinks( } ow( - options, + options as any, ow.object.exactShape({ urls: ow.array.ofType(ow.string), requestQueue: ow.object.hasKeys('addRequestsBatched'), diff --git a/packages/core/src/enqueue_links/index.ts b/packages/core/src/enqueue_links/index.ts index d650fd270c33..3582f2a5eb7d 100644 --- a/packages/core/src/enqueue_links/index.ts +++ b/packages/core/src/enqueue_links/index.ts @@ -1,2 +1,2 @@ -export * from './enqueue_links'; -export * from './shared'; +export * from './enqueue_links.js'; +export * from './shared.js'; diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts index eae7603135b2..20690991e469 100644 --- a/packages/core/src/enqueue_links/shared.ts +++ b/packages/core/src/enqueue_links/shared.ts @@ -5,9 +5,9 @@ import { Minimatch } from 'minimatch'; import { purlToRegExp } from '@apify/pseudo_url'; -import type { RequestOptions } from '../request'; -import { Request } from '../request'; -import type { EnqueueLinksOptions } from './enqueue_links'; +import type { RequestOptions } from '../request.js'; +import { Request } from '../request.js'; +import type { EnqueueLinksOptions } from './enqueue_links.js'; export { tryAbsoluteURL } from '@crawlee/utils'; @@ -240,7 +240,7 @@ export function filterRequestsByPatterns( * @ignore */ export function createRequestOptions( - sources: (string | Record)[], + sources: readonly (string | Record)[], options: Pick = {}, ): RequestOptions[] { return sources diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts index 3e55610caf62..a0f95d740e24 100644 --- a/packages/core/src/errors.ts +++ b/packages/core/src/errors.ts @@ -35,3 +35,27 @@ export class SessionError extends RetryRequestError { super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`); } } + +export class ContextPipelineInterruptedError extends Error { + constructor(message?: string) { + super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`); + } +} + +export class ContextPipelineInitializationError extends Error { + constructor(error: unknown, options?: ErrorOptions) { + super(undefined, { cause: error, ...options }); + } +} + +export class ContextPipelineCleanupError extends CriticalError { + constructor(error: unknown, options?: ErrorOptions) { + super(undefined, { cause: error, ...options }); + } +} + +export class RequestHandlerError extends Error { + constructor(error: unknown, options?: ErrorOptions) { + super(undefined, { cause: error, ...options }); + } +} diff --git a/packages/core/src/events/event_manager.ts b/packages/core/src/events/event_manager.ts index c8cad080a41e..e134974ec581 100644 --- a/packages/core/src/events/event_manager.ts +++ b/packages/core/src/events/event_manager.ts @@ -4,7 +4,7 @@ import log from '@apify/log'; import type { BetterIntervalID } from '@apify/utilities'; import { betterClearInterval, betterSetInterval } from '@apify/utilities'; -import { Configuration } from '../configuration'; +import { Configuration } from '../configuration.js'; export const enum EventType { PERSIST_STATE = 'persistState', diff --git a/packages/core/src/events/index.ts b/packages/core/src/events/index.ts index 8e8144c469cb..211d9af2a79f 100644 --- a/packages/core/src/events/index.ts +++ b/packages/core/src/events/index.ts @@ -1,2 +1,2 @@ -export * from './event_manager'; -export * from './local_event_manager'; +export * from './event_manager.js'; +export * from './local_event_manager.js'; diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.ts index 27ca2eeb7c15..6df0bb1ff4b7 100644 --- a/packages/core/src/events/local_event_manager.ts +++ b/packages/core/src/events/local_event_manager.ts @@ -1,16 +1,12 @@ -import os from 'node:os'; - -import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils'; +import { getCurrentCpuTicksV2, getMemoryInfo, isContainerized } from '@crawlee/utils'; import log from '@apify/log'; import { betterClearInterval, betterSetInterval } from '@apify/utilities'; -import type { SystemInfo } from '../autoscaling'; -import { EventManager, EventType } from './event_manager'; +import type { SystemInfo } from '../autoscaling/system_status.js'; +import { EventManager, EventType } from './event_manager.js'; export class LocalEventManager extends EventManager { - private previousTicks = { idle: 0, total: 0 }; - /** * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events. * This is automatically called at the beginning of `crawler.run()`. @@ -57,20 +53,6 @@ export class LocalEventManager extends EventManager { return this.config.get('containerized', await isContainerized()); } - private getCurrentCpuTicks() { - const cpus = os.cpus(); - return cpus.reduce( - (acc, cpu) => { - const cpuTimes = Object.values(cpu.times); - return { - idle: acc.idle + cpu.times.idle, - total: acc.total + cpuTimes.reduce((sum, num) => sum + num), - }; - }, - { idle: 0, total: 0 }, - ); - } - /** * Creates a SystemInfo object based on local metrics. */ @@ -83,19 +65,7 @@ export class LocalEventManager extends EventManager { } private async createCpuInfo(options: { maxUsedCpuRatio: number }) { - if (this.config.get('systemInfoV2')) { - const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper()); - return { - cpuCurrentUsage: usedCpuRatio * 100, - isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio, - }; - } - const ticks = this.getCurrentCpuTicks(); - const idleTicksDelta = ticks.idle - this.previousTicks!.idle; - const totalTicksDelta = ticks.total - this.previousTicks!.total; - const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0; - Object.assign(this.previousTicks, ticks); - + const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper()); return { cpuCurrentUsage: usedCpuRatio * 100, isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio, @@ -104,13 +74,7 @@ export class LocalEventManager extends EventManager { private async createMemoryInfo() { try { - if (this.config.get('systemInfoV2')) { - const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper()); - return { - memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes, - }; - } - const memInfo = await getMemoryInfo(); + const memInfo = await getMemoryInfo(await this.isContainerizedWrapper()); return { memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes, }; diff --git a/packages/core/src/http_clients/base-http-client.ts b/packages/core/src/http_clients/base-http-client.ts deleted file mode 100644 index 94491c27fafb..000000000000 --- a/packages/core/src/http_clients/base-http-client.ts +++ /dev/null @@ -1,239 +0,0 @@ -import type { Readable } from 'node:stream'; - -import { applySearchParams, type SearchParams } from '@crawlee/utils'; - -import type { FormDataLike } from './form-data-like'; - -type Timeout = - | { - lookup: number; - connect: number; - secureConnect: number; - socket: number; - send: number; - response: number; - } - | { request: number }; - -type Method = - | 'GET' - | 'POST' - | 'PUT' - | 'PATCH' - | 'HEAD' - | 'DELETE' - | 'OPTIONS' - | 'TRACE' - | 'get' - | 'post' - | 'put' - | 'patch' - | 'head' - | 'delete' - | 'options' - | 'trace'; - -/** - * Maps permitted values of the `responseType` option on {@apilink HttpRequest} to the types that they produce. - */ -export interface ResponseTypes { - 'json': unknown; - 'text': string; - 'buffer': Buffer; -} - -interface Progress { - percent: number; - transferred: number; - total?: number; -} - -// TODO BC with got - remove the options and callback parameters in 4.0 -interface ToughCookieJar { - getCookieString: (( - currentUrl: string, - options: Record, - callback: (error: Error | null, cookies: string) => void, - ) => string) & - ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string); - setCookie: (( - cookieOrString: unknown, - currentUrl: string, - options: Record, - callback: (error: Error | null, cookie: unknown) => void, - ) => void) & - ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void); -} - -interface PromiseCookieJar { - getCookieString: (url: string) => Promise; - setCookie: (rawCookie: string, url: string) => Promise; -} - -type SimpleHeaders = Record; - -/** - * HTTP Request as accepted by {@apilink BaseHttpClient} methods. - */ -export interface HttpRequest { - [k: string]: unknown; // TODO BC with got - remove in 4.0 - - url: string | URL; - method?: Method; - headers?: SimpleHeaders; - body?: string | Buffer | Readable | Generator | AsyncGenerator | FormDataLike; - - signal?: AbortSignal; - timeout?: Partial; - - cookieJar?: ToughCookieJar | PromiseCookieJar; - followRedirect?: boolean | ((response: any) => boolean); // TODO BC with got - specify type better in 4.0 - maxRedirects?: number; - - encoding?: BufferEncoding; - responseType?: TResponseType; - throwHttpErrors?: boolean; - - // from got-scraping Context - proxyUrl?: string; - headerGeneratorOptions?: Record; - useHeaderGenerator?: boolean; - headerGenerator?: { - getHeaders: (options: Record) => Record; - }; - insecureHTTPParser?: boolean; - sessionToken?: object; -} - -/** - * Additional options for HTTP requests that need to be handled separately before passing to {@apilink BaseHttpClient}. - */ -export interface HttpRequestOptions - extends HttpRequest { - /** Search (query string) parameters to be appended to the request URL */ - searchParams?: SearchParams; - - /** A form to be sent in the HTTP request body (URL encoding will be used) */ - form?: Record; - /** Artbitrary object to be JSON-serialized and sent as the HTTP request body */ - json?: unknown; - - /** Basic HTTP Auth username */ - username?: string; - /** Basic HTTP Auth password */ - password?: string; -} - -/** - * HTTP response data, without a body, as returned by {@apilink BaseHttpClient} methods. - */ -export interface BaseHttpResponseData { - redirectUrls: URL[]; - url: string; - - ip?: string; - statusCode: number; - statusMessage?: string; - - headers: SimpleHeaders; - trailers: SimpleHeaders; // Populated after the whole message is processed - - complete: boolean; -} - -interface HttpResponseWithoutBody - extends BaseHttpResponseData { - request: HttpRequest; -} - -/** - * HTTP response data as returned by the {@apilink BaseHttpClient.sendRequest} method. - */ -export interface HttpResponse - extends HttpResponseWithoutBody { - [k: string]: any; // TODO BC with got - remove in 4.0 - - body: ResponseTypes[TResponseType]; -} - -/** - * HTTP response data as returned by the {@apilink BaseHttpClient.stream} method. - */ -export interface StreamingHttpResponse extends HttpResponseWithoutBody { - stream: Readable; - readonly downloadProgress: Progress; - readonly uploadProgress: Progress; -} - -/** - * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument. - */ -export type RedirectHandler = ( - redirectResponse: BaseHttpResponseData, - updatedRequest: { url?: string | URL; headers: SimpleHeaders }, -) => void; - -/** - * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl. - */ -export interface BaseHttpClient { - /** - * Perform an HTTP Request and return the complete response. - */ - sendRequest( - request: HttpRequest, - ): Promise>; - - /** - * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response. - */ - stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise; -} - -/** - * Converts {@apilink HttpRequestOptions} to a {@apilink HttpRequest}. - */ -export function processHttpRequestOptions({ - searchParams, - form, - json, - username, - password, - ...request -}: HttpRequestOptions): HttpRequest { - const url = new URL(request.url); - const headers = { ...request.headers }; - - applySearchParams(url, searchParams); - - if ([request.body, form, json].filter((value) => value !== undefined).length > 1) { - throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments'); - } - - const body = (() => { - if (form !== undefined) { - return new URLSearchParams(form).toString(); - } - - if (json !== undefined) { - return JSON.stringify(json); - } - - return request.body; - })(); - - if (form !== undefined) { - headers['content-type'] ??= 'application/x-www-form-urlencoded'; - } - - if (json !== undefined) { - headers['content-type'] ??= 'application/json'; - } - - if (username !== undefined || password !== undefined) { - const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64'); - headers.authorization = `Basic ${encodedAuth}`; - } - - return { ...request, body, url, headers }; -} diff --git a/packages/core/src/http_clients/form-data-like.ts b/packages/core/src/http_clients/form-data-like.ts deleted file mode 100644 index 784bd960ac8e..000000000000 --- a/packages/core/src/http_clients/form-data-like.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * This is copied from https://github.com/octet-stream/form-data-encoder - */ - -interface FileLike { - /** - * Name of the file referenced by the File object. - */ - readonly name: string; - /** - * Returns the media type ([`MIME`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types)) of the file represented by a `File` object. - */ - readonly type: string; - /** - * Size of the file parts in bytes - */ - readonly size: number; - /** - * The last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). Files without a known last modified date return the current date. - */ - readonly lastModified: number; - /** - * Returns a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) which upon reading returns the data contained within the [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File). - */ - stream(): ReadableStream | AsyncIterable; - readonly [Symbol.toStringTag]?: string; -} - -/** - * A `string` or `File` that represents a single value from a set of `FormData` key-value pairs. - */ -type FormDataEntryValue = string | FileLike; -/** - * This interface reflects minimal shape of the FormData - */ -export interface FormDataLike { - /** - * Appends a new value onto an existing key inside a FormData object, - * or adds the key if it does not already exist. - * - * The difference between `set()` and `append()` is that if the specified key already exists, `set()` will overwrite all existing values with the new one, whereas `append()` will append the new value onto the end of the existing set of values. - * - * @param name The name of the field whose data is contained in `value`. - * @param value The field's value. This can be [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) - or [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File). If none of these are specified the value is converted to a string. - * @param fileName The filename reported to the server, when a Blob or File is passed as the second parameter. The default filename for Blob objects is "blob". The default filename for File objects is the file's filename. - */ - append(name: string, value: unknown, fileName?: string): void; - /** - * Returns all the values associated with a given key from within a `FormData` object. - * - * @param {string} name A name of the value you want to retrieve. - * - * @returns An array of `FormDataEntryValue` whose key matches the value passed in the `name` parameter. If the key doesn't exist, the method returns an empty list. - */ - getAll(name: string): FormDataEntryValue[]; - /** - * Returns an [`iterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols) allowing to go through the `FormData` key/value pairs. - * The key of each pair is a string; the value is a [`FormDataValue`](https://developer.mozilla.org/en-US/docs/Web/API/FormDataEntryValue). - */ - entries(): IterableIterator<[string, FormDataEntryValue]>; - /** - * An alias for FormDataLike#entries() - */ - [Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]>; - readonly [Symbol.toStringTag]?: string; -} diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.ts deleted file mode 100644 index be75c6dafb08..000000000000 --- a/packages/core/src/http_clients/got-scraping-http-client.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { gotScraping } from '@crawlee/utils'; -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { Options, PlainResponse } from 'got-scraping'; - -import type { - BaseHttpClient, - HttpRequest, - HttpResponse, - RedirectHandler, - ResponseTypes, - StreamingHttpResponse, -} from './base-http-client'; - -/** - * A HTTP client implementation based on the `got-scraping` library. - */ -export class GotScrapingHttpClient implements BaseHttpClient { - /** - * @inheritDoc - */ - async sendRequest( - request: HttpRequest, - ): Promise> { - const gotResult = await gotScraping({ - ...request, - // `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`. - // Using the `cookieJar` option directly would override that. - cookieJar: undefined, - retry: { - limit: 0, - ...(request.retry as Record | undefined), - }, - }); - - return { - ...gotResult, - body: gotResult.body as ResponseTypes[TResponseType], - request: { url: request.url, ...gotResult.request }, - }; - } - - /** - * @inheritDoc - */ - async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise { - // eslint-disable-next-line no-async-promise-executor - return new Promise(async (resolve, reject) => { - const stream = await Promise.resolve(gotScraping({ ...request, isStream: true, cookieJar: undefined })); - - stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => { - handleRedirect?.(redirectResponse, updatedOptions); - }); - - // We need to end the stream for DELETE requests, otherwise it will hang. - if (request.method && ['DELETE', 'delete'].includes(request.method)) { - stream.end(); - } - - stream.on('error', reject); - - stream.on('response', (response: PlainResponse) => { - const result: StreamingHttpResponse = { - stream, - request, - redirectUrls: response.redirectUrls, - url: response.url, - ip: response.ip, - statusCode: response.statusCode, - headers: response.headers, - trailers: response.trailers, - complete: response.complete, - get downloadProgress() { - return stream.downloadProgress; - }, - get uploadProgress() { - return stream.uploadProgress; - }, - }; - - Object.assign(result, response); // TODO BC - remove in 4.0 - - resolve(result); - - stream.on('end', () => { - result.complete = response.complete; - - result.trailers ??= {}; - Object.assign(result.trailers, response.trailers); - - (result as any).rawTrailers ??= []; // TODO BC - remove in 4.0 - Object.assign((result as any).rawTrailers, response.rawTrailers); - }); - }); - }); - } -} diff --git a/packages/core/src/http_clients/index.ts b/packages/core/src/http_clients/index.ts deleted file mode 100644 index 58c1b27a5313..000000000000 --- a/packages/core/src/http_clients/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './base-http-client'; -export * from './got-scraping-http-client'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ee0625f0c69a..1f327e0e6ccd 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,19 +1,18 @@ -export * from './errors'; -export * from './autoscaling'; -export * from './configuration'; -export * from './crawlers'; -export * from './enqueue_links'; -export * from './events'; -export * from './http_clients'; -export * from './log'; -export * from './proxy_configuration'; -export * from './request'; -export * from './router'; -export * from './serialization'; -export * from './session_pool'; -export * from './storages'; -export * from './validators'; -export * from './cookie_utils'; -export * from './recoverable_state'; +export * from './errors.js'; +export * from './autoscaling/index.js'; +export * from './configuration.js'; +export * from './crawlers/index.js'; +export * from './enqueue_links/index.js'; +export * from './events/index.js'; +export * from './log.js'; +export * from './proxy_configuration.js'; +export * from './request.js'; +export * from './router.js'; +export * from './serialization.js'; +export * from './session_pool/index.js'; +export * from './storages/index.js'; +export * from './validators.js'; +export * from './cookie_utils.js'; +export * from './recoverable_state.js'; export { PseudoUrl } from '@apify/pseudo_url'; export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types'; diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.ts index 132d923aca83..b16f8de04104 100644 --- a/packages/core/src/proxy_configuration.ts +++ b/packages/core/src/proxy_configuration.ts @@ -1,13 +1,12 @@ -import type { Dictionary } from '@crawlee/types'; +import type { Dictionary, ProxyInfo } from '@crawlee/types'; import ow from 'ow'; import log from '@apify/log'; -import { cryptoRandomObjectId } from '@apify/utilities'; -import type { Request } from './request'; +import type { Request } from './request.js'; export interface ProxyConfigurationFunction { - (sessionId: string | number, options?: { request?: Request }): string | null | Promise; + (options?: { request?: Request }): string | null | Promise; } type UrlList = (string | null)[]; @@ -21,7 +20,7 @@ export interface ProxyConfigurationOptions { proxyUrls?: UrlList; /** - * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter and an optional parameter with the `Request` object when applicable. + * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable. * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous. * * This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called. @@ -47,73 +46,6 @@ export interface TieredProxy { proxyTier?: number; } -/** - * The main purpose of the ProxyInfo object is to provide information - * about the current proxy connection used by the crawler for the request. - * Outside of crawlers, you can get this object by calling {@apilink ProxyConfiguration.newProxyInfo}. - * - * **Example usage:** - * - * ```javascript - * const proxyConfiguration = new ProxyConfiguration({ - * proxyUrls: ['...', '...'] // List of Proxy URLs to rotate - * }); - * - * // Getting proxyInfo object by calling class method directly - * const proxyInfo = await proxyConfiguration.newProxyInfo(); - * - * // In crawler - * const crawler = new CheerioCrawler({ - * // ... - * proxyConfiguration, - * requestHandler({ proxyInfo }) { - * // Getting used proxy URL - * const proxyUrl = proxyInfo.url; - * - * // Getting ID of used Session - * const sessionIdentifier = proxyInfo.sessionId; - * } - * }) - * - * ``` - */ -export interface ProxyInfo { - /** - * The identifier of used {@apilink Session}, if used. - */ - sessionId?: string; - - /** - * The URL of the proxy. - */ - url: string; - - /** - * Username for the proxy. - */ - username?: string; - - /** - * User's password for the proxy. - */ - password: string; - - /** - * Hostname of your proxy. - */ - hostname: string; - - /** - * Proxy port. - */ - port: number | string; - - /** - * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`). - */ - proxyTier?: number; -} - interface TieredProxyOptions { request?: Request; proxyTier?: number; @@ -260,28 +192,18 @@ export class ProxyConfiguration { * the currently used proxy via the requestHandler parameter `proxyInfo`. * Use it if you want to work with a rich representation of a proxy URL. * If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}. - * @param [sessionId] - * Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or - * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier. - * When the provided sessionId is a number, it's converted to a string. Property sessionId of - * {@apilink ProxyInfo} is always returned as a type string. * - * All the HTTP requests going through the proxy with the same session identifier - * will use the same target proxy server (i.e. the same IP address). - * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`. * @return Represents information about used proxy and its configuration. */ - async newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise { - if (typeof sessionId === 'number') sessionId = `${sessionId}`; - + async newProxyInfo(options?: TieredProxyOptions): Promise { let url: string | undefined; let tier: number | undefined; if (this.tieredProxyUrls) { - const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options); + const { proxyUrl, proxyTier } = this._handleTieredUrl(options); url = proxyUrl ?? undefined; tier = proxyTier; } else { - url = await this.newUrl(sessionId, options); + url = await this.newUrl(options); } if (!url) return undefined; @@ -289,7 +211,6 @@ export class ProxyConfiguration { const { username, password, port, hostname } = new URL(url); return { - sessionId, url, username: decodeURIComponent(username), password: decodeURIComponent(password), @@ -300,12 +221,11 @@ export class ProxyConfiguration { } /** - * Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options. - * @param _sessionId Session identifier + * Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options. * @param options Options for the tiered proxy rotation * @returns An object with the proxy URL and the proxy tier used. */ - protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): TieredProxy { + protected _handleTieredUrl(options?: TieredProxyOptions): TieredProxy { if (!this.tieredProxyUrls) throw new Error('Tiered proxy URLs are not set'); if (!options || (!options?.request && options?.proxyTier === undefined)) { @@ -368,57 +288,32 @@ export class ProxyConfiguration { } /** - * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter. - * @param [sessionId] - * Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or - * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier. - * When the provided sessionId is a number, it's converted to a string. + * Returns a new proxy URL based on provided configuration options. * - * All the HTTP requests going through the proxy with the same session identifier - * will use the same target proxy server (i.e. the same IP address). - * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`. * @return A string with a proxy URL, including authentication credentials and port number. * For example, `http://bob:password123@proxy.example.com:8000` */ - async newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise { - if (typeof sessionId === 'number') sessionId = `${sessionId}`; - + async newUrl(options?: TieredProxyOptions): Promise { if (this.newUrlFunction) { - return (await this._callNewUrlFunction(sessionId, { request: options?.request })) ?? undefined; + return (await this._callNewUrlFunction({ request: options?.request })) ?? undefined; } if (this.tieredProxyUrls) { - return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined; + return this._handleTieredUrl(options).proxyUrl ?? undefined; } - return this._handleCustomUrl(sessionId) ?? undefined; + return this._handleProxyUrlsList() ?? undefined; } - /** - * Handles custom url rotation with session - */ - protected _handleCustomUrl(sessionId?: string): string | null { - let customUrlToUse: string | null; - - if (!sessionId) { - return this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length]; - } - - if (this.usedProxyUrls.has(sessionId)) { - customUrlToUse = this.usedProxyUrls.get(sessionId)!; - } else { - customUrlToUse = this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length]; - this.usedProxyUrls.set(sessionId, customUrlToUse); - } - - return customUrlToUse; + protected _handleProxyUrlsList(): string | null { + return this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length]; } /** * Calls the custom newUrlFunction and checks format of its return value */ - protected async _callNewUrlFunction(sessionId?: string, options?: { request?: Request }) { - const proxyUrl = await this.newUrlFunction!(sessionId!, options); + protected async _callNewUrlFunction(options?: { request?: Request }) { + const proxyUrl = await this.newUrlFunction!(options); try { if (proxyUrl) { new URL(proxyUrl); // eslint-disable-line no-new diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts index bafc6a1d2188..28c79709607a 100644 --- a/packages/core/src/request.ts +++ b/packages/core/src/request.ts @@ -8,11 +8,11 @@ import ow from 'ow'; import { normalizeUrl } from '@apify/utilities'; -import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links'; -import type { SkippedRequestReason } from './enqueue_links/shared'; -import { log as defaultLog } from './log'; -import type { AllowedHttpMethods } from './typedefs'; -import { keys } from './typedefs'; +import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js'; +import type { SkippedRequestReason } from './enqueue_links/shared.js'; +import { log as defaultLog } from './log.js'; +import type { AllowedHttpMethods } from './typedefs.js'; +import { keys } from './typedefs.js'; // new properties on the Request object breaks serialization const log = defaultLog.child({ prefix: 'Request' }); @@ -81,7 +81,7 @@ export enum RequestState { * ``` * @category Sources */ -export class Request { +class CrawleeRequest { /** Request ID */ id?: string; @@ -196,7 +196,8 @@ export class Request { this.url = url; this.loadedUrl = loadedUrl; this.uniqueKey = - uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }); + uniqueKey || + CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }); this.method = method; this.payload = payload; this.noRetry = noRetry; @@ -259,6 +260,18 @@ export class Request { } } + /** + * Converts the Crawlee Request object to a `fetch` API Request object. + * @returns The native `fetch` API Request object. + */ + public intoFetchAPIRequest(): Request { + return new Request(this.url, { + method: this.method, + headers: this.headers, + body: this.payload, + }); + } + /** Tells the crawler processing this request to skip the navigation and process the request directly. */ get skipNavigation(): boolean { return this.userData.__crawlee?.skipNavigation ?? false; @@ -399,16 +412,6 @@ export class Request { this.errorMessages.push(message); } - // TODO: only for better BC, remove in v4 - protected _computeUniqueKey(options: ComputeUniqueKeyOptions) { - return Request.computeUniqueKey(options); - } - - // TODO: only for better BC, remove in v4 - protected _hashPayload(payload: BinaryLike): string { - return Request.hashPayload(payload); - } - /** @internal */ static computeUniqueKey({ url, @@ -421,8 +424,7 @@ export class Request { const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors. if (!useExtendedUniqueKey) { if (normalizedMethod !== 'GET' && payload) { - // Using log.deprecated to log only once. We should add log.once or some such. - log.deprecated( + log.warningOnce( `We've encountered a ${normalizedMethod} Request with a payload. ` + 'This is fine. Just letting you know that if your requests point to the same URL ' + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.', @@ -430,7 +432,7 @@ export class Request { } return normalizedUrl; } - const payloadHash = payload ? Request.hashPayload(payload) : ''; + const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : ''; return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`; } @@ -572,10 +574,12 @@ interface ComputeUniqueKeyOptions { useExtendedUniqueKey?: boolean; } -export type Source = (Partial & { requestsFromUrl?: string; regex?: RegExp }) | Request; +export type Source = (Partial & { requestsFromUrl?: string; regex?: RegExp }) | CrawleeRequest; /** @internal */ export interface InternalSource { requestsFromUrl: string; regex?: RegExp; } + +export { CrawleeRequest as Request }; diff --git a/packages/core/src/router.ts b/packages/core/src/router.ts index 545bb3360db8..25a17b503d64 100644 --- a/packages/core/src/router.ts +++ b/packages/core/src/router.ts @@ -1,9 +1,9 @@ import type { Dictionary } from '@crawlee/types'; -import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons'; -import { MissingRouteError } from './errors'; -import type { Request } from './request'; -import type { Awaitable } from './typedefs'; +import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js'; +import { MissingRouteError } from './errors.js'; +import type { Request } from './request.js'; +import type { Awaitable } from './typedefs.js'; const defaultRoute = Symbol('default-route'); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index ace72f2068e5..fe7b7fdfc693 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -3,7 +3,7 @@ import util from 'node:util'; import zlib from 'node:zlib'; import ow from 'ow'; -import StreamArray from 'stream-json/streamers/StreamArray'; +import StreamArray from 'stream-json/streamers/StreamArray.js'; const pipeline = util.promisify(streamPipeline); @@ -102,12 +102,8 @@ export function createDeserialize(compressedData: Buffer | Uint8Array): Readable const streamArray = StreamArray.withParser(); const destination = pluckValue(streamArray); - streamPipeline( - Readable.from([compressedData]), - zlib.createGunzip(), - destination, - // @ts-expect-error Something's wrong here, the types are wrong but tests fail if we correct the code to make them right - (err) => destination.emit(err), + streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err: any) => + destination.emit(err), ); return destination; diff --git a/packages/core/src/session_pool/index.ts b/packages/core/src/session_pool/index.ts index eaedabfa4981..e9b25291682a 100644 --- a/packages/core/src/session_pool/index.ts +++ b/packages/core/src/session_pool/index.ts @@ -1,5 +1,5 @@ -export * from './errors'; -export * from './events'; -export * from './session'; -export * from './session_pool'; -export * from './consts'; +export * from './errors.js'; +export * from './events.js'; +export * from './session.js'; +export * from './session_pool.js'; +export * from './consts.js'; diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts index 180dbe8a9652..95c5cd7378f2 100644 --- a/packages/core/src/session_pool/session.ts +++ b/packages/core/src/session_pool/session.ts @@ -1,38 +1,21 @@ import { EventEmitter } from 'node:events'; -import type { Cookie as CookieObject, Dictionary } from '@crawlee/types'; +import type { Cookie as CookieObject, Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types'; import ow from 'ow'; -import type { Cookie, SerializedCookieJar } from 'tough-cookie'; +import type { Cookie } from 'tough-cookie'; import { CookieJar } from 'tough-cookie'; import type { Log } from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; -import type { ResponseLike } from '../cookie_utils'; import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, -} from '../cookie_utils'; -import { log as defaultLog } from '../log'; -import { EVENT_SESSION_RETIRED } from './events'; - -/** - * Persistable {@apilink Session} state. - */ -export interface SessionState { - id: string; - cookieJar: SerializedCookieJar; - userData: object; - errorScore: number; - maxErrorScore: number; - errorScoreDecrement: number; - usageCount: number; - maxUsageCount: number; - expiresAt: string; - createdAt: string; -} +} from '../cookie_utils.js'; +import { log as defaultLog } from '../log.js'; +import { EVENT_SESSION_RETIRED } from './events.js'; export interface SessionOptions { /** Id of session used for generating fingerprints. It is used as proxy session name. */ @@ -84,11 +67,12 @@ export interface SessionOptions { maxUsageCount?: number; /** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */ - sessionPool?: import('./session_pool').SessionPool; + sessionPool?: import('./session_pool.js').SessionPool; log?: Log; errorScore?: number; cookieJar?: CookieJar; + proxyInfo?: ProxyInfo; } /** @@ -97,7 +81,7 @@ export interface SessionOptions { * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general. * @category Scaling */ -export class Session { +export class Session implements ISession { readonly id: string; private maxAgeSecs: number; userData: Dictionary; @@ -107,8 +91,9 @@ export class Session { private _expiresAt: Date; private _usageCount: number; private _maxUsageCount: number; - private sessionPool: import('./session_pool').SessionPool; + private sessionPool: import('./session_pool.js').SessionPool; private _errorScore: number; + private _proxyInfo?: ProxyInfo; private _cookieJar: CookieJar; private log: Log; @@ -144,6 +129,10 @@ export class Session { return this._cookieJar; } + get proxyInfo() { + return this._proxyInfo; + } + /** * Session configuration. */ @@ -154,6 +143,7 @@ export class Session { sessionPool: ow.object.instanceOf(EventEmitter), id: ow.optional.string, cookieJar: ow.optional.object, + proxyInfo: ow.optional.object, maxAgeSecs: ow.optional.number, userData: ow.optional.object, maxErrorScore: ow.optional.number, @@ -171,6 +161,7 @@ export class Session { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), + proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, @@ -187,6 +178,7 @@ export class Session { this.log = log.child({ prefix: 'Session' }); this._cookieJar = (cookieJar.setCookie as unknown) ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar)); + this._proxyInfo = proxyInfo; this.id = id; this.maxAgeSecs = maxAgeSecs; this.userData = userData; @@ -257,6 +249,7 @@ export class Session { return { id: this.id, cookieJar: this.cookieJar.toJSON()!, + proxyInfo: this._proxyInfo, userData: this.userData, maxErrorScore: this.maxErrorScore, errorScoreDecrement: this.errorScoreDecrement, @@ -298,33 +291,14 @@ export class Session { /** * With certain status codes: `401`, `403` or `429` we can be certain * that the target website is blocking us. This function helps to do this conveniently - * by retiring the session when such code is received. Optionally the default status - * codes can be extended in the second parameter. - * @param statusCode HTTP status code. - * @returns Whether the session was retired. - */ - retireOnBlockedStatusCodes(statusCode: number): boolean; - - /** - * With certain status codes: `401`, `403` or `429` we can be certain - * that the target website is blocking us. This function helps to do this conveniently - * by retiring the session when such code is received. Optionally the default status + * by retiring the session when such code is received. Optionally, the default status * codes can be extended in the second parameter. * @param statusCode HTTP status code. - * @param [additionalBlockedStatusCodes] - * Custom HTTP status codes that means blocking on particular website. - * - * **This parameter is deprecated and will be removed in next major version.** * @returns Whether the session was retired. - * @deprecated The parameter `additionalBlockedStatusCodes` is deprecated and will be removed in next major version. */ - retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean; - - retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes: number[] = []): boolean { + retireOnBlockedStatusCodes(statusCode: number): boolean { // eslint-disable-next-line dot-notation -- accessing private property - const isBlocked = this.sessionPool['blockedStatusCodes'] - .concat(additionalBlockedStatusCodes) - .includes(statusCode); + const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode); if (isBlocked) { this.retire(); } @@ -338,10 +312,10 @@ export class Session { * * It then parses and saves the cookies from the `set-cookie` header, if available. */ - setCookiesFromResponse(response: ResponseLike) { + setCookiesFromResponse(response: Response) { try { const cookies = getCookiesFromResponse(response).filter((c) => c); - this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url!); + this._setCookies(cookies, response.url); } catch (e) { const err = e as Error; // if invalid Cookie header is provided just log the exception. diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts index 22528a29ebbd..8ade797b8ea3 100644 --- a/packages/core/src/session_pool/session_pool.ts +++ b/packages/core/src/session_pool/session_pool.ts @@ -6,15 +6,15 @@ import ow from 'ow'; import type { Log } from '@apify/log'; -import { Configuration } from '../configuration'; -import type { PersistenceOptions } from '../crawlers/statistics'; -import type { EventManager } from '../events/event_manager'; -import { EventType } from '../events/event_manager'; -import { log as defaultLog } from '../log'; -import { KeyValueStore } from '../storages/key_value_store'; -import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts'; -import type { SessionOptions } from './session'; -import { Session } from './session'; +import { Configuration } from '../configuration.js'; +import type { PersistenceOptions } from '../crawlers/statistics.js'; +import type { EventManager } from '../events/event_manager.js'; +import { EventType } from '../events/event_manager.js'; +import { log as defaultLog } from '../log.js'; +import { KeyValueStore } from '../storages/key_value_store.js'; +import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js'; +import type { SessionOptions } from './session.js'; +import { Session } from './session.js'; /** * Factory user-function which creates customized {@apilink Session} instances. @@ -47,7 +47,7 @@ export interface SessionPoolOptions { persistStateKey?: string; /** - * Custom function that should return `Session` instance. + * Custom function that should return a `Session` instance, or a promise resolving to such instance. * Any error thrown from this function will terminate the process. * Function receives `SessionPool` instance as a parameter */ @@ -282,6 +282,21 @@ export class SessionPool extends EventEmitter { this._addSession(newSession); } + /** + * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool, + * but this allows you to add more sessions once the max pool size is reached. + * This also allows you to add session with overridden session options (e.g. with specific session id). + * @param [options] The configuration options for the session being added to the session pool. + */ + async newSession(sessionOptions?: SessionOptions): Promise { + this._throwIfNotInitialized(); + + const newSession = await this.createSessionFunction(this, { sessionOptions }); + this._addSession(newSession); + + return newSession; + } + /** * Gets session. * If there is space for new session, it creates and returns new session. @@ -434,12 +449,13 @@ export class SessionPool extends EventEmitter { * @param [options.sessionOptions] The configuration options for the session being created. * @returns New session. */ - protected _defaultCreateSessionFunction( + protected async _defaultCreateSessionFunction( sessionPool: SessionPool, options: { sessionOptions?: SessionOptions } = {}, - ): Session { + ): Promise { ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object })); const { sessionOptions = {} } = options; + return new Session({ ...this.sessionOptions, ...sessionOptions, diff --git a/packages/core/src/storages/access_checking.ts b/packages/core/src/storages/access_checking.ts index 941823e8db37..c56612a2c70d 100644 --- a/packages/core/src/storages/access_checking.ts +++ b/packages/core/src/storages/access_checking.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks'; -import type { Awaitable } from '../typedefs'; +import type { Awaitable } from '../typedefs.js'; const storage = new AsyncLocalStorage<{ checkFunction: () => void }>(); diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts index dc27d09b381c..8b5f97db1903 100644 --- a/packages/core/src/storages/dataset.ts +++ b/packages/core/src/storages/dataset.ts @@ -4,14 +4,14 @@ import ow from 'ow'; import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts'; -import { Configuration } from '../configuration'; -import { type Log, log } from '../log'; -import type { Awaitable } from '../typedefs'; -import { checkStorageAccess } from './access_checking'; -import { KeyValueStore } from './key_value_store'; -import type { StorageManagerOptions } from './storage_manager'; -import { StorageManager } from './storage_manager'; -import { purgeDefaultStorages } from './utils'; +import { Configuration } from '../configuration.js'; +import { type Log, log } from '../log.js'; +import type { Awaitable } from '../typedefs.js'; +import { checkStorageAccess } from './access_checking.js'; +import { KeyValueStore } from './key_value_store.js'; +import type { StorageManagerOptions } from './storage_manager.js'; +import { StorageManager } from './storage_manager.js'; +import { purgeDefaultStorages } from './utils.js'; /** @internal */ export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000; diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts index ebe9eb2ea528..46e3813984c6 100644 --- a/packages/core/src/storages/index.ts +++ b/packages/core/src/storages/index.ts @@ -1,13 +1,13 @@ -export * from './dataset'; -export * from './key_value_store'; -export * from './request_list'; -export * from './request_list_adapter'; -export * from './request_provider'; -export { RequestQueueV1 } from './request_queue'; -export { RequestQueue } from './request_queue_v2'; -export { RequestQueue as RequestQueueV2 } from './request_queue_v2'; -export * from './storage_manager'; -export * from './utils'; -export * from './access_checking'; -export * from './sitemap_request_list'; -export * from './request_manager_tandem'; +export * from './dataset.js'; +export * from './key_value_store.js'; +export * from './request_list.js'; +export * from './request_list_adapter.js'; +export * from './request_provider.js'; +export { RequestQueueV1 } from './request_queue.js'; +export { RequestQueue } from './request_queue_v2.js'; +export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js'; +export * from './storage_manager.js'; +export * from './utils.js'; +export * from './access_checking.js'; +export * from './sitemap_request_list.js'; +export * from './request_manager_tandem.js'; diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts index 161d4fa29a05..b710ccab380c 100644 --- a/packages/core/src/storages/key_value_store.ts +++ b/packages/core/src/storages/key_value_store.ts @@ -9,12 +9,12 @@ import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts'; import log from '@apify/log'; import { jsonStringifyExtended } from '@apify/utilities'; -import { Configuration } from '../configuration'; -import type { Awaitable } from '../typedefs'; -import { checkStorageAccess } from './access_checking'; -import type { StorageManagerOptions } from './storage_manager'; -import { StorageManager } from './storage_manager'; -import { purgeDefaultStorages } from './utils'; +import { Configuration } from '../configuration.js'; +import type { Awaitable } from '../typedefs.js'; +import { checkStorageAccess } from './access_checking.js'; +import type { StorageManagerOptions } from './storage_manager.js'; +import { StorageManager } from './storage_manager.js'; +import { purgeDefaultStorages } from './utils.js'; /** * Helper function to possibly stringify value if options.contentType is not set. @@ -483,10 +483,13 @@ export class KeyValueStore { /** * Returns a file URL for the given key. + * + * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`. + * + * @param key The key of the record to generate the public URL for. */ - getPublicUrl(key: string): string { - const name = this.name ?? this.config.get('defaultKeyValueStoreId'); - return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`; + async getPublicUrl(key: string): Promise { + return this.client.getRecordPublicUrl(key); } /** diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts index 4e6a09741103..f4dd457f3f9f 100644 --- a/packages/core/src/storages/request_list.ts +++ b/packages/core/src/storages/request_list.ts @@ -1,16 +1,16 @@ -import type { Dictionary } from '@crawlee/types'; +import type { BaseHttpClient, Dictionary } from '@crawlee/types'; import { downloadListOfUrls } from '@crawlee/utils'; import ow, { ArgumentError } from 'ow'; -import { Configuration } from '../configuration'; -import type { EventManager } from '../events'; -import { EventType } from '../events'; -import { log } from '../log'; -import type { ProxyConfiguration } from '../proxy_configuration'; -import { type InternalSource, Request, type RequestOptions, type Source } from '../request'; -import { createDeserialize, serializeArray } from '../serialization'; -import { KeyValueStore } from './key_value_store'; -import { purgeDefaultStorages } from './utils'; +import { Configuration } from '../configuration.js'; +import type { EventManager } from '../events/event_manager.js'; +import { EventType } from '../events/event_manager.js'; +import { log } from '../log.js'; +import type { ProxyConfiguration } from '../proxy_configuration.js'; +import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js'; +import { createDeserialize, serializeArray } from '../serialization.js'; +import { KeyValueStore } from './key_value_store.js'; +import { purgeDefaultStorages } from './utils.js'; /** @internal */ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE'; @@ -234,6 +234,13 @@ export interface RequestListOptions { /** @internal */ config?: Configuration; + + /** + * The HTTP client to be used to download `requestsFromUrl` URLs. + * + * If not specified the `RequestList` will use the default HTTP client. + */ + httpClient?: BaseHttpClient; } /** @@ -348,6 +355,7 @@ export class RequestList implements IRequestList { private sourcesFunction?: RequestListSourcesFunction; private proxyConfiguration?: ProxyConfiguration; private events: EventManager; + private httpClient?: BaseHttpClient; /** * To create new instance of `RequestList` we need to use `RequestList.open()` factory method. @@ -364,6 +372,7 @@ export class RequestList implements IRequestList { proxyConfiguration, keepDuplicateUrls = false, config = Configuration.getGlobalConfig(), + httpClient, } = options; if (!(sources || sourcesFunction)) { @@ -386,6 +395,7 @@ export class RequestList implements IRequestList { }), keepDuplicateUrls: ow.optional.boolean, proxyConfiguration: ow.optional.object, + httpClient: ow.optional.object, }), ); @@ -393,6 +403,7 @@ export class RequestList implements IRequestList { this.persistRequestsKey = persistRequestsKey ? `SDK_${persistRequestsKey}` : persistRequestsKey; this.initialState = state; this.events = config.getEventManager(); + this.httpClient = httpClient; // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list. this.keepDuplicateUrls = keepDuplicateUrls; @@ -967,7 +978,10 @@ export class RequestList implements IRequestList { urlRegExp?: RegExp; proxyUrl?: string; }): Promise { - return downloadListOfUrls(options); + return downloadListOfUrls({ + ...options, + httpClient: this.httpClient, + }); } } diff --git a/packages/core/src/storages/request_list_adapter.ts b/packages/core/src/storages/request_list_adapter.ts index 0e39dea17a3e..363622955124 100644 --- a/packages/core/src/storages/request_list_adapter.ts +++ b/packages/core/src/storages/request_list_adapter.ts @@ -1,13 +1,13 @@ import type { Dictionary } from '@crawlee/types'; -import type { Request } from '../request'; -import type { IRequestList } from './request_list'; +import type { Request } from '../request.js'; +import type { IRequestList } from './request_list.js'; import type { AddRequestsBatchedResult, IRequestManager, RequestQueueOperationInfo, RequestQueueOperationOptions, -} from './request_provider'; +} from './request_provider.js'; /** * Adapts the IRequestList interface to the IRequestManager interface. diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.ts index cc79cf45c4f6..0d1ad21ff32d 100644 --- a/packages/core/src/storages/request_manager_tandem.ts +++ b/packages/core/src/storages/request_manager_tandem.ts @@ -2,9 +2,9 @@ import type { Dictionary } from '@crawlee/types'; import type { Log } from '@apify/log'; -import { log } from '../log'; -import type { Request, Source } from '../request'; -import type { IRequestList } from './request_list'; +import { log } from '../log.js'; +import type { Request, Source } from '../request.js'; +import type { IRequestList } from './request_list.js'; import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, @@ -12,7 +12,7 @@ import type { RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike, -} from './request_provider'; +} from './request_provider.js'; /** * A request manager that combines a RequestList and a RequestQueue. diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts index dc1204abf5aa..b16c9c38ab8e 100644 --- a/packages/core/src/storages/request_provider.ts +++ b/packages/core/src/storages/request_provider.ts @@ -1,6 +1,7 @@ import { inspect } from 'node:util'; import type { + BaseHttpClient, BatchAddRequestsResult, Dictionary, ProcessedRequest, @@ -19,22 +20,23 @@ import { sleep, } from '@crawlee/utils'; import ow from 'ow'; +import type { ReadonlyDeep } from 'type-fest'; import { ListDictionary, LruCache } from '@apify/datastructures'; import type { Log } from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; -import { Configuration } from '../configuration'; -import { EventType } from '../events'; -import { log } from '../log'; -import type { ProxyConfiguration } from '../proxy_configuration'; -import type { InternalSource, RequestOptions, Source } from '../request'; -import { Request } from '../request'; -import type { Constructor } from '../typedefs'; -import { checkStorageAccess } from './access_checking'; -import type { IStorage, StorageManagerOptions } from './storage_manager'; -import { StorageManager } from './storage_manager'; -import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils'; +import { Configuration } from '../configuration.js'; +import { EventType } from '../events/event_manager.js'; +import { log } from '../log.js'; +import type { ProxyConfiguration } from '../proxy_configuration.js'; +import type { InternalSource, RequestOptions, Source } from '../request.js'; +import { Request } from '../request.js'; +import type { Constructor } from '../typedefs.js'; +import { checkStorageAccess } from './access_checking.js'; +import type { IStorage, StorageManagerOptions } from './storage_manager.js'; +import { StorageManager } from './storage_manager.js'; +import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js'; export type RequestsLike = AsyncIterable | Iterable | (Source | string)[]; @@ -132,6 +134,8 @@ export abstract class RequestProvider implements IStorage, IRequestManager { protected inProgressRequestBatchCount = 0; + protected httpClient?: BaseHttpClient; + constructor( options: InternalRequestProviderOptions, readonly config = Configuration.getGlobalConfig(), @@ -393,7 +397,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager { * @param options Options for the request queue */ async addRequestsBatched( - requests: RequestsLike, + requests: ReadonlyDeep, options: AddRequestsBatchedOptions = {}, ): Promise { checkStorageAccess(); @@ -837,7 +841,10 @@ export abstract class RequestProvider implements IStorage, IRequestManager { urlRegExp?: RegExp; proxyUrl?: string; }): Promise { - return downloadListOfUrls(options); + return downloadListOfUrls({ + ...options, + httpClient: this.httpClient, + }); } /** @@ -866,6 +873,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager { config: ow.optional.object.instanceOf(Configuration), storageClient: ow.optional.object, proxyConfiguration: ow.optional.object, + httpClient: ow.optional.object, }), ); @@ -882,6 +890,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager { queue.initialCount = queueInfo?.totalRequestCount ?? 0; queue.initialHandledCount = queueInfo?.handledRequestCount ?? 0; + queue.httpClient = options.httpClient; return queue; } diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts index 804b18739991..01c0a8b0bb54 100644 --- a/packages/core/src/storages/request_queue.ts +++ b/packages/core/src/storages/request_queue.ts @@ -4,11 +4,11 @@ import type { Dictionary } from '@crawlee/types'; import { REQUEST_QUEUE_HEAD_MAX_LIMIT } from '@apify/consts'; -import { Configuration } from '../configuration'; -import type { Request } from '../request'; -import { checkStorageAccess } from './access_checking'; -import type { RequestProviderOptions, RequestQueueOperationInfo } from './request_provider'; -import { RequestProvider } from './request_provider'; +import { Configuration } from '../configuration.js'; +import type { Request } from '../request.js'; +import { checkStorageAccess } from './access_checking.js'; +import type { RequestProviderOptions, RequestQueueOperationInfo } from './request_provider.js'; +import { RequestProvider } from './request_provider.js'; import { API_PROCESSED_REQUESTS_DELAY_MILLIS, getRequestId, @@ -16,7 +16,7 @@ import { QUERY_HEAD_BUFFER, QUERY_HEAD_MIN_LENGTH, STORAGE_CONSISTENCY_DELAY_MILLIS, -} from './utils'; +} from './utils.js'; const MAX_CACHED_REQUESTS = 1_000_000; diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.ts index 7dd8157d7ca0..74b156aada47 100644 --- a/packages/core/src/storages/request_queue_v2.ts +++ b/packages/core/src/storages/request_queue_v2.ts @@ -1,17 +1,17 @@ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types'; -import { Configuration } from '../configuration'; -import { EventType } from '../events'; -import type { Request, Source } from '../request'; -import { checkStorageAccess } from './access_checking'; +import { Configuration } from '../configuration.js'; +import { EventType } from '../events/event_manager.js'; +import type { Request, Source } from '../request.js'; +import { checkStorageAccess } from './access_checking.js'; import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike, -} from './request_provider'; -import { RequestProvider } from './request_provider'; -import { getRequestId } from './utils'; +} from './request_provider.js'; +import { RequestProvider } from './request_provider.js'; +import { getRequestId } from './utils.js'; // Double the limit of RequestQueue v1 (1_000_000) as we also store keyed by request.id, not just from uniqueKey const MAX_CACHED_REQUESTS = 2_000_000; diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.ts index 87e06bca8675..6fbe0b455c8a 100644 --- a/packages/core/src/storages/sitemap_request_list.ts +++ b/packages/core/src/storages/sitemap_request_list.ts @@ -1,5 +1,6 @@ import { Transform } from 'node:stream'; +import type { BaseHttpClient } from '@crawlee/types'; import { parseSitemap, type ParseSitemapOptions } from '@crawlee/utils'; import { minimatch } from 'minimatch'; import ow from 'ow'; @@ -7,14 +8,14 @@ import type { RequiredDeep } from 'type-fest'; import defaultLog from '@apify/log'; -import { Configuration } from '../configuration'; -import type { GlobInput, RegExpInput, UrlPatternObject } from '../enqueue_links'; -import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links'; -import { type EventManager, EventType } from '../events/event_manager'; -import { Request } from '../request'; -import { KeyValueStore } from './key_value_store'; -import type { IRequestList } from './request_list'; -import { purgeDefaultStorages } from './utils'; +import { Configuration } from '../configuration.js'; +import type { GlobInput, RegExpInput, UrlPatternObject } from '../enqueue_links/shared.js'; +import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links/shared.js'; +import { type EventManager, EventType } from '../events/event_manager.js'; +import { Request } from '../request.js'; +import { KeyValueStore } from './key_value_store.js'; +import type { IRequestList } from './request_list.js'; +import { purgeDefaultStorages } from './utils.js'; /** @internal */ const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LIST_STATE'; @@ -103,6 +104,10 @@ export interface SitemapRequestListOptions extends UrlConstraints { * Crawlee configuration */ config?: Configuration; + /** + * Custom HTTP client to be used for sitemap loading. + */ + httpClient?: BaseHttpClient; } interface SitemapParsingProgress { @@ -190,7 +195,7 @@ export class SitemapRequestList implements IRequestList { /** * Proxy URL to be used for sitemap loading. */ - private proxyUrl: string | undefined; + private proxyUrl?: string; /** * Logger instance. @@ -414,12 +419,14 @@ export class SitemapRequestList implements IRequestList { * Track the loading progress using the `isSitemapFullyLoaded` property. */ static async open(options: SitemapRequestListOptions): Promise { + const { httpClient, ...restOptions } = options; + const requestList = new SitemapRequestList({ - ...options, + ...restOptions, persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY, }); await requestList.restoreState(); - void requestList.load({ parseSitemapOptions: options.parseSitemapOptions }); + void requestList.load({ parseSitemapOptions: { ...options.parseSitemapOptions, httpClient } }); if (requestList.persistenceOptions.enable) { requestList.events.on(EventType.PERSIST_STATE, requestList.persistState); diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts index aea40468c9ca..66ec619cd76d 100644 --- a/packages/core/src/storages/storage_manager.ts +++ b/packages/core/src/storages/storage_manager.ts @@ -1,9 +1,9 @@ -import type { Dictionary, StorageClient } from '@crawlee/types'; +import type { BaseHttpClient, Dictionary, StorageClient } from '@crawlee/types'; import { AsyncQueue } from '@sapphire/async-queue'; -import { Configuration } from '../configuration'; -import type { ProxyConfiguration } from '../proxy_configuration'; -import type { Constructor } from '../typedefs'; +import { Configuration } from '../configuration.js'; +import type { ProxyConfiguration } from '../proxy_configuration.js'; +import type { Constructor } from '../typedefs.js'; const DEFAULT_ID_CONFIG_KEYS = { Dataset: 'defaultDatasetId', @@ -172,4 +172,9 @@ export interface StorageManagerOptions { * If undefined, the `requestsFromUrl` requests will be made without proxy. */ proxyConfiguration?: ProxyConfiguration; + + /** + * HTTP client to be used to download the list of URLs in `RequestQueue`. + */ + httpClient?: BaseHttpClient; } diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts index 31135c948dd7..b6820e7562ec 100644 --- a/packages/core/src/storages/utils.ts +++ b/packages/core/src/storages/utils.ts @@ -2,8 +2,8 @@ import crypto from 'node:crypto'; import type { Dictionary, StorageClient } from '@crawlee/types'; -import { Configuration } from '../configuration'; -import { KeyValueStore } from './key_value_store'; +import { Configuration } from '../configuration.js'; +import { KeyValueStore } from './key_value_store.js'; /** * Options for purging default storage. diff --git a/packages/core/src/typedefs.ts b/packages/core/src/typedefs.ts index 49f7f49c1d2a..9564cda5fa86 100644 --- a/packages/core/src/typedefs.ts +++ b/packages/core/src/typedefs.ts @@ -14,13 +14,4 @@ export function keys(obj: T) { return Object.keys(obj) as (keyof T)[]; } -export declare type AllowedHttpMethods = - | 'GET' - | 'HEAD' - | 'POST' - | 'PUT' - | 'DELETE' - | 'TRACE' - | 'OPTIONS' - | 'CONNECT' - | 'PATCH'; +export { AllowedHttpMethods } from '@crawlee/types'; diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json index 6da76a20bad7..0b6e99562af3 100644 --- a/packages/crawlee/package.json +++ b/packages/crawlee/package.json @@ -1,20 +1,14 @@ { "name": "crawlee", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, "bin": "./src/cli.ts", - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -47,27 +41,27 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@crawlee/basic": "3.15.3", - "@crawlee/browser": "3.15.3", - "@crawlee/browser-pool": "3.15.3", - "@crawlee/cheerio": "3.15.3", - "@crawlee/cli": "3.15.3", - "@crawlee/core": "3.15.3", - "@crawlee/http": "3.15.3", - "@crawlee/jsdom": "3.15.3", - "@crawlee/linkedom": "3.15.3", - "@crawlee/playwright": "3.15.3", - "@crawlee/puppeteer": "3.15.3", - "@crawlee/utils": "3.15.3", - "import-local": "^3.1.0", - "tslib": "^2.4.0" + "@crawlee/basic": "4.0.0", + "@crawlee/browser": "4.0.0", + "@crawlee/browser-pool": "4.0.0", + "@crawlee/cheerio": "4.0.0", + "@crawlee/cli": "4.0.0", + "@crawlee/core": "4.0.0", + "@crawlee/http": "4.0.0", + "@crawlee/jsdom": "4.0.0", + "@crawlee/linkedom": "4.0.0", + "@crawlee/playwright": "4.0.0", + "@crawlee/puppeteer": "4.0.0", + "@crawlee/utils": "4.0.0", + "import-local": "^3.2.0", + "tslib": "^2.8.1" }, "peerDependencies": { "idcac-playwright": "*", diff --git a/packages/crawlee/src/cli.ts b/packages/crawlee/src/cli.ts index e776789f5cc1..d1aa2f9f0653 100755 --- a/packages/crawlee/src/cli.ts +++ b/packages/crawlee/src/cli.ts @@ -1,9 +1,8 @@ #!/usr/bin/env node -// eslint-disable-next-line -const importLocal = require('import-local'); +import importLocal from 'import-local'; -if (!importLocal(__filename)) { - // eslint-disable-next-line - require('@crawlee/cli'); +// @ts-ignore bad types most likely? +if (!importLocal(import.meta.url)) { + await import('@crawlee/cli'); } diff --git a/packages/got-scraping-client/README.md b/packages/got-scraping-client/README.md new file mode 100644 index 000000000000..68fd8c2fc770 --- /dev/null +++ b/packages/got-scraping-client/README.md @@ -0,0 +1,28 @@ +# @crawlee/got-scraping-client + +This package provides a Crawlee-compliant `HttpClient` interface for the [`got-scraping`](https://www.npmjs.com/package/got-scraping) package. + +To use the `got-scraping` package directly without Crawlee, check out [`got-scraping`](https://www.npmjs.com/package/got-scraping) on NPM. + +## Example usage + +Simply pass the `GotScrapingHttpClient` instance to the `httpClient` option of the crawler constructor: + +```typescript +import { CheerioCrawler, Dictionary } from '@crawlee/cheerio'; +import { GotScrapingHttpClient, Browser } from '@crawlee/got-scraping-client'; + +const crawler = new CheerioCrawler({ + httpClient: new GotScrapingHttpClient(), + async requestHandler({ $, request }) { + // Extract the title of the page. + const title = $('title').text(); + console.log(`Title of the page ${request.url}: ${title}`); + }, +}); + +crawler.run([ + 'http://www.example.com/page-1', + 'http://www.example.com/page-2', +]); +``` diff --git a/packages/got-scraping-client/package.json b/packages/got-scraping-client/package.json new file mode 100644 index 000000000000..3d156b33f735 --- /dev/null +++ b/packages/got-scraping-client/package.json @@ -0,0 +1,53 @@ +{ + "name": "@crawlee/got-scraping-client", + "version": "4.0.0", + "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", + "engines": { + "node": ">=22.0.0" + }, + "type": "module", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "keywords": [ + "apify", + "headless", + "chrome", + "puppeteer", + "crawler", + "scraper" + ], + "author": { + "name": "Apify", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [ + "Jan Curn ", + "Marek Trunkat ", + "Ondra Urban " + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/apify/crawlee" + }, + "bugs": { + "url": "https://github.com/apify/crawlee/issues" + }, + "homepage": "https://crawlee.dev", + "scripts": { + "build": "yarn clean && yarn compile && yarn copy", + "clean": "rimraf ./dist", + "compile": "tsc -p tsconfig.build.json", + "copy": "tsx ../../scripts/copy.ts" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@crawlee/http-client": "4.0.0", + "got-scraping": "^4.1.3" + } +} diff --git a/packages/got-scraping-client/src/index.ts b/packages/got-scraping-client/src/index.ts new file mode 100644 index 000000000000..298c47024189 --- /dev/null +++ b/packages/got-scraping-client/src/index.ts @@ -0,0 +1,63 @@ +import { Readable } from 'node:stream'; + +import { BaseHttpClient, type CustomFetchOptions, ResponseWithUrl } from '@crawlee/http-client'; +import { gotScraping, type Options } from 'got-scraping'; + +/** + * A HTTP client implementation based on the `got-scraping` library. + */ +export class GotScrapingHttpClient extends BaseHttpClient { + /** + * Type guard that validates the HTTP method (excluding CONNECT). + * @param request - The HTTP request to validate + */ + private validateRequest( + request: Request, + ): request is Request & { method: Exclude } { + return !['CONNECT', 'connect'].includes(request.method!); + } + + private *iterateHeaders( + headers: Record, + ): Generator<[string, string], void, unknown> { + for (const [key, value] of Object.entries(headers)) { + if (key.startsWith(':') || value === undefined) continue; + if (Array.isArray(value)) { + for (const v of value) yield [key, v]; + } else { + yield [key, value]; + } + } + } + + private parseHeaders(headers: Record): Headers { + return new Headers([...this.iterateHeaders(headers)]); + } + + override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise { + const { proxyUrl, redirect } = options ?? {}; + + if (!this.validateRequest(request)) { + throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`); + } + + const gotResult = await gotScraping({ + url: request.url!, + method: request.method as Options['method'], + headers: Object.fromEntries(request.headers.entries()), + body: request.body ? Readable.fromWeb(request.body as any) : undefined, + proxyUrl, + signal: options?.signal ?? undefined, + followRedirect: redirect === 'follow', + }); + + const responseHeaders = this.parseHeaders(gotResult.headers); + + return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), { + headers: responseHeaders, + status: gotResult.statusCode, + statusText: gotResult.statusMessage ?? '', + url: gotResult.url, + }); + } +} diff --git a/packages/got-scraping-client/tsconfig.build.json b/packages/got-scraping-client/tsconfig.build.json new file mode 100644 index 000000000000..9bc5ad54c68b --- /dev/null +++ b/packages/got-scraping-client/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/packages/got-scraping-client/tsconfig.json b/packages/got-scraping-client/tsconfig.json new file mode 100644 index 000000000000..2e6a4ce4084f --- /dev/null +++ b/packages/got-scraping-client/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/packages/http-client/package.json b/packages/http-client/package.json new file mode 100644 index 000000000000..e7d4c5b4f3f0 --- /dev/null +++ b/packages/http-client/package.json @@ -0,0 +1,53 @@ +{ + "name": "@crawlee/http-client", + "version": "4.0.0", + "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", + "engines": { + "node": ">=22.0.0" + }, + "type": "module", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "keywords": [ + "apify", + "headless", + "chrome", + "puppeteer", + "crawler", + "scraper" + ], + "author": { + "name": "Apify", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [ + "Jan Curn ", + "Marek Trunkat ", + "Ondra Urban " + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/apify/crawlee" + }, + "bugs": { + "url": "https://github.com/apify/crawlee/issues" + }, + "homepage": "https://crawlee.dev", + "scripts": { + "build": "yarn clean && yarn compile && yarn copy", + "clean": "rimraf ./dist", + "compile": "tsc -p tsconfig.build.json", + "copy": "tsx ../../scripts/copy.ts" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@crawlee/types": "4.0.0", + "tough-cookie": "^6.0.0" + } +} diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts new file mode 100644 index 000000000000..f2831f0af598 --- /dev/null +++ b/packages/http-client/src/base-http-client.ts @@ -0,0 +1,119 @@ +import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types'; +import { CookieJar } from 'tough-cookie'; + +export interface CustomFetchOptions { + proxyUrl?: string; +} + +/** + * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed + * behaviors (redirect handling, proxy and cookie handling). Concrete clients + * implement only the low-level network call in `fetch`. + */ +export abstract class BaseHttpClient implements BaseHttpClientInterface { + /** + * Perform the raw network request and return a single Response without any + * automatic redirect following or special error handling. + */ + protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise; + + private async applyCookies(request: Request, cookieJar: CookieJar): Promise { + const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean); + + if (cookies?.length > 0) { + request.headers.set('cookie', cookies.join('; ')); + } + return request; + } + + private async setCookies(response: Response, cookieJar: CookieJar): Promise { + const setCookieHeaders = response.headers.getSetCookie(); + + await Promise.all(setCookieHeaders.map((header) => cookieJar.setCookie(header, response.url))); + } + + private resolveRequestContext(options?: SendRequestOptions): { + proxyUrl?: string; + cookieJar: CookieJar; + timeout?: number; + } { + const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url; + const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar(); + const timeout = options?.timeout; + return { proxyUrl, cookieJar: cookieJar as CookieJar, timeout }; + } + + private createAbortSignal(timeout?: number): AbortSignal | undefined { + return timeout ? AbortSignal.timeout(timeout) : undefined; + } + + private isRedirect(response: Response): boolean { + const status = response.status; + return status >= 300 && status < 400 && !!response.headers.get('location'); + } + + private buildRedirectRequest(currentRequest: Request, response: Response, initialRequest: Request): Request { + const location = response.headers.get('location')!; + const nextUrl = new URL(location, response.url ?? currentRequest.url); + + const prevMethod = (currentRequest.method ?? 'GET').toUpperCase(); + let nextMethod = prevMethod; + let nextBody: BodyInit | null = null; + + if ( + response.status === 303 || + ((response.status === 301 || response.status === 302) && prevMethod === 'POST') + ) { + nextMethod = 'GET'; + nextBody = null; + } else { + const clonedRequest = initialRequest.clone(); + nextBody = clonedRequest.body; + } + + const nextHeaders = new Headers(); + currentRequest.headers.forEach((value, key) => nextHeaders.set(key, value)); + + return new Request(nextUrl.toString(), { + method: nextMethod, + headers: nextHeaders, + body: nextBody, + credentials: (currentRequest as any).credentials, + redirect: 'manual', + }); + } + + /** + * Public fetch-like method that handles redirects and uses provided proxy and cookie jar. + */ + async sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise { + const maxRedirects = 10; + let currentRequest = initialRequest; + let redirectCount = 0; + + const { proxyUrl, cookieJar, timeout } = this.resolveRequestContext(options); + currentRequest = initialRequest.clone(); + + while (true) { + await this.applyCookies(currentRequest, cookieJar); + + const response = await this.fetch(currentRequest, { + signal: this.createAbortSignal(timeout), + proxyUrl, + redirect: 'manual', + }); + + await this.setCookies(response, cookieJar); + + if (this.isRedirect(response)) { + if (redirectCount++ >= maxRedirects) { + throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`); + } + currentRequest = this.buildRedirectRequest(currentRequest, response, initialRequest); + continue; + } + + return response; + } + } +} diff --git a/packages/http-client/src/fetch-http-client.ts b/packages/http-client/src/fetch-http-client.ts new file mode 100644 index 000000000000..48395390f3b1 --- /dev/null +++ b/packages/http-client/src/fetch-http-client.ts @@ -0,0 +1,12 @@ +import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js'; + +/** + * A HTTP client implementation using the native `fetch` API. + * + * This implementation does not support proxying. + */ +export class FetchHttpClient extends BaseHttpClient { + override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise { + return fetch(request, options); + } +} diff --git a/packages/http-client/src/index.ts b/packages/http-client/src/index.ts new file mode 100644 index 000000000000..104f8fbf5057 --- /dev/null +++ b/packages/http-client/src/index.ts @@ -0,0 +1,3 @@ +export { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js'; +export { ResponseWithUrl, type IResponseWithUrl } from './response.js'; +export { FetchHttpClient } from './fetch-http-client.js'; diff --git a/packages/http-client/src/response.ts b/packages/http-client/src/response.ts new file mode 100644 index 000000000000..15268b0f3392 --- /dev/null +++ b/packages/http-client/src/response.ts @@ -0,0 +1,21 @@ +export interface IResponseWithUrl extends Response { + url: string; +} + +// See https://github.com/nodejs/undici/blob/d7707ee8fd5da2d0cc64b5fae421b965faf803c8/lib/web/fetch/constants.js#L6 +const nullBodyStatus = [101, 204, 205, 304]; + +/** + * A Response class that includes the original request URL. + * + * This class extends `Response` from `fetch` API and is fully compatible with this. + */ +export class ResponseWithUrl extends Response implements IResponseWithUrl { + override url: string; + constructor(body: BodyInit | null, init: ResponseInit & { url?: string }) { + const bodyParsed = nullBodyStatus.includes(init.status ?? 200) ? null : body; + + super(bodyParsed, init); + this.url = init.url ?? ''; + } +} diff --git a/packages/http-client/tsconfig.build.json b/packages/http-client/tsconfig.build.json new file mode 100644 index 000000000000..9bc5ad54c68b --- /dev/null +++ b/packages/http-client/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/packages/http-client/tsconfig.json b/packages/http-client/tsconfig.json new file mode 100644 index 000000000000..2e6a4ce4084f --- /dev/null +++ b/packages/http-client/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json index de93248c68d6..6cb9ce155ed7 100644 --- a/packages/http-crawler/package.json +++ b/packages/http-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/http", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,26 +40,26 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.7.10", - "@crawlee/basic": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "@types/content-type": "^1.1.5", - "cheerio": "1.0.0-rc.12", - "content-type": "^1.0.4", - "got-scraping": "^4.0.0", + "@apify/timeout": "^0.3.2", + "@apify/utilities": "^2.15.5", + "@crawlee/basic": "4.0.0", + "@crawlee/http-client": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "@types/content-type": "^1.1.8", + "cheerio": "^1.0.0", + "content-type": "^1.0.5", "iconv-lite": "^0.7.0", - "mime-types": "^2.1.35", - "ow": "^0.28.1", - "tslib": "^2.4.0", - "type-fest": "^4.0.0" + "mime-types": "^3.0.1", + "ow": "^2.0.0", + "tslib": "^2.8.1", + "type-fest": "^4.41.0" } } diff --git a/packages/http-crawler/src/index.ts b/packages/http-crawler/src/index.ts index 26b3ec966179..b81749842f81 100644 --- a/packages/http-crawler/src/index.ts +++ b/packages/http-crawler/src/index.ts @@ -1,3 +1,3 @@ export * from '@crawlee/basic'; -export * from './internals/http-crawler'; -export * from './internals/file-download'; +export * from './internals/http-crawler.js'; +export * from './internals/file-download.js'; diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts index 536a96681deb..704a459f2888 100644 --- a/packages/http-crawler/src/internals/file-download.ts +++ b/packages/http-crawler/src/internals/file-download.ts @@ -1,63 +1,34 @@ import { Transform } from 'node:stream'; import { finished } from 'node:stream/promises'; -import { isPromise } from 'node:util/types'; +import type { BasicCrawlerOptions } from '@crawlee/basic'; +import { BasicCrawler, ContextPipeline } from '@crawlee/basic'; +import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core'; import type { Dictionary } from '@crawlee/types'; -// @ts-expect-error got-scraping is ESM only -import type { Request } from 'got-scraping'; -import type { - ErrorHandler, - GetUserDataFromRequest, - HttpCrawlerOptions, - InternalHttpCrawlingContext, - InternalHttpHook, - RequestHandler, - RouterRoutes, -} from '../index'; -import { HttpCrawler, Router } from '../index'; +import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js'; +import { Router } from '../index.js'; +import { parseContentTypeFromResponse } from './utils.js'; export type FileDownloadErrorHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler - JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> = ErrorHandler>; - -export type StreamHandlerContext = Omit< - FileDownloadCrawlingContext, - 'body' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType' -> & { - stream: Request; // TODO BC - remove in v4 -}; - -type StreamHandler = (context: StreamHandlerContext) => void | Promise; - -export type FileDownloadOptions< - UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler - JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> = - | (Omit>, 'requestHandler'> & { - requestHandler?: never; - streamHandler?: StreamHandler; - }) - | (Omit>, 'requestHandler'> & { - requestHandler: FileDownloadRequestHandler; - streamHandler?: never; - }); +> = ErrorHandler>; export type FileDownloadHook< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler - JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> = InternalHttpHook>; +> = InternalHttpHook>; export interface FileDownloadCrawlingContext< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler - JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends InternalHttpCrawlingContext {} +> extends CrawlingContext { + request: LoadedRequest>; + response: Response; + contentType: { type: string; encoding: BufferEncoding }; +} export type FileDownloadRequestHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler - JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> = RequestHandler>; +> = RequestHandler>; /** * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed. @@ -156,11 +127,11 @@ export function ByteCounterStream({ * * The crawler finishes when there are no more {@apilink Request} objects to crawl. * - * We can use the `preNavigationHooks` to adjust `gotOptions`: + * We can use the `preNavigationHooks` to adjust the crawling context before the request is made: * * ``` * preNavigationHooks: [ - * (crawlingContext, gotOptions) => { + * (crawlingContext) => { * // ... * }, * ] @@ -184,99 +155,37 @@ export function ByteCounterStream({ * ]); * ``` */ -export class FileDownload extends HttpCrawler { - private streamHandler?: StreamHandler; - - constructor(options: FileDownloadOptions = {}) { - const { streamHandler } = options; - delete options.streamHandler; - - if (streamHandler) { - // For streams, the navigation is done in the request handler. - (options as any).requestHandlerTimeoutSecs = options.navigationTimeoutSecs ?? 120; - } - - super(options); - - this.streamHandler = streamHandler; - if (this.streamHandler) { - this.requestHandler = this.streamRequestHandler as any; - } - - // The base HttpCrawler class only supports a handful of text based mime types. - // With the FileDownload crawler, we want to download any file type. - (this as any).supportedMimeTypes = new Set(['*/*']); - } - - protected override async _runRequestHandler(context: FileDownloadCrawlingContext) { - if (this.streamHandler) { - context.request.skipNavigation = true; - } - - await super._runRequestHandler(context); +export class FileDownload extends BasicCrawler { + // TODO hooks + constructor(options: BasicCrawlerOptions = {}) { + super({ + ...options, + contextPipelineBuilder: () => + ContextPipeline.create().compose({ + action: async (context) => this.initiateDownload(context), + cleanup: async (context) => { + await (context.response.body ? finished(context.response.body as any) : Promise.resolve()); + }, + }), + }); } - private async streamRequestHandler(context: FileDownloadCrawlingContext) { - const { - log, - request: { url }, - } = context; - - const response = await this.httpClient.stream({ - url, - timeout: { request: undefined }, - proxyUrl: context.proxyInfo?.url, + private async initiateDownload(context: CrawlingContext) { + const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), { + session: context.session, }); - let pollingInterval: NodeJS.Timeout | undefined; - - const cleanUp = () => { - clearInterval(pollingInterval!); - response.stream.destroy(); - }; - - const downloadPromise = new Promise((resolve, reject) => { - pollingInterval = setInterval(() => { - const { total, transferred } = response.downloadProgress; - - if (transferred > 0) { - log.debug(`Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${url}.`); - } - }, 5000); + const { type, charset: encoding } = parseContentTypeFromResponse(response); - response.stream.on('error', async (error: Error) => { - cleanUp(); - reject(error); - }); + context.request.url = response.url; - let streamHandlerResult; - - try { - context.stream = response.stream; - context.response = response as any; - streamHandlerResult = this.streamHandler!(context as any); - } catch (e) { - cleanUp(); - reject(e); - } - - if (isPromise(streamHandlerResult)) { - streamHandlerResult - .then(() => { - resolve(); - }) - .catch((e: Error) => { - cleanUp(); - reject(e); - }); - } else { - resolve(); - } - }); - - await Promise.all([downloadPromise, finished(response.stream)]); + const contextExtension = { + request: context.request as LoadedRequest, + response, + contentType: { type, encoding }, + }; - cleanUp(); + return contextExtension; } } diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts index 4be576749138..3cebb1eec3b7 100644 --- a/packages/http-crawler/src/internals/http-crawler.ts +++ b/packages/http-crawler/src/internals/http-crawler.ts @@ -1,6 +1,4 @@ -import type { IncomingHttpHeaders, IncomingMessage } from 'node:http'; -import { extname } from 'node:path'; -import type { Readable } from 'node:stream'; +import { Readable } from 'node:stream'; import util from 'node:util'; import type { @@ -9,52 +7,36 @@ import type { CrawlingContext, ErrorHandler, GetUserDataFromRequest, - LoadedContext, - ProxyConfiguration, - Request, + Request as CrawleeRequest, RequestHandler, + RequireContextPipeline, RouterRoutes, Session, } from '@crawlee/basic'; import { - BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES, Configuration, - CrawlerExtension, + ContextPipeline, mergeCookies, - processHttpRequestOptions, RequestState, Router, SessionError, - validators, } from '@crawlee/basic'; -import type { HttpResponse, StreamingHttpResponse } from '@crawlee/core'; +import type { LoadedRequest } from '@crawlee/core'; +import { ResponseWithUrl } from '@crawlee/http-client'; import type { Awaitable, Dictionary } from '@crawlee/types'; import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils'; import * as cheerio from 'cheerio'; import type { RequestLike, ResponseLike } from 'content-type'; import contentTypeParser from 'content-type'; -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { Method, OptionsInit, TimeoutError as TimeoutErrorClass } from 'got-scraping'; import iconv from 'iconv-lite'; -import mime from 'mime-types'; -import ow, { ObjectPredicate } from 'ow'; +import ow from 'ow'; import type { JsonValue } from 'type-fest'; import { addTimeoutToPromise, tryCancel } from '@apify/timeout'; -import { concatStreamToBuffer, readStreamToString } from '@apify/utilities'; -let TimeoutError: typeof TimeoutErrorClass; - -/** - * TODO exists for BC within HttpCrawler - replace completely with StreamingHttpResponse in 4.0 - * @internal - */ -export type PlainResponse = Omit & - IncomingMessage & { - body?: unknown; - }; +import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js'; /** * Default mime types, which HttpScraper supports. @@ -77,15 +59,11 @@ export type HttpErrorHandler< JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler > = ErrorHandler>; -export interface HttpCrawlerOptions - extends BasicCrawlerOptions { - /** - * An alias for {@apilink HttpCrawlerOptions.requestHandler} - * Soon to be removed, use `requestHandler` instead. - * @deprecated - */ - handlePageFunction?: HttpCrawlerOptions['requestHandler']; - +export interface HttpCrawlerOptions< + Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, +> extends BasicCrawlerOptions { /** * Timeout in which the HTTP request to the resource needs to finish, given in seconds. */ @@ -96,21 +74,14 @@ export interface HttpCrawlerOptions { + * async (crawlingContext) => { * // ... * }, * ] @@ -119,7 +90,7 @@ export interface HttpCrawlerOptions[]; + preNavigationHooks?: InternalHttpHook[]; /** * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. @@ -133,7 +104,7 @@ export interface HttpCrawlerOptions[]; + postNavigationHooks?: ((crawlingContext: CrawlingContextWithReponse) => Awaitable)[]; /** * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types) @@ -189,21 +160,34 @@ export interface HttpCrawlerOptions = (crawlingContext: Context, gotOptions: OptionsInit) => Awaitable; +export type InternalHttpHook = (crawlingContext: Context) => Awaitable; export type HttpHook< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler > = InternalHttpHook>; +interface CrawlingContextWithReponse< + UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler +> extends CrawlingContext { + /** + * The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property. + */ + request: LoadedRequest>; + + /** + * The HTTP response object containing status code, headers, and other response metadata. + */ + response: Response; +} + /** * @internal */ export interface InternalHttpCrawlingContext< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler - Crawler = HttpCrawler, -> extends CrawlingContext { +> extends CrawlingContextWithReponse { /** * The request body of the web page. * The type depends on the `Content-Type` header of the web page: @@ -221,7 +205,6 @@ export interface InternalHttpCrawlingContext< * Parsed `Content-Type header: { type, encoding }`. */ contentType: { type: string; encoding: BufferEncoding }; - response: PlainResponse; /** * Wait for an element matching the selector to appear. Timeout is ignored. @@ -253,7 +236,7 @@ export interface InternalHttpCrawlingContext< } export interface HttpCrawlingContext - extends InternalHttpCrawlingContext>> {} + extends InternalHttpCrawlingContext {} export type HttpRequestHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler @@ -281,11 +264,11 @@ export type HttpRequestHandler< * * The crawler finishes when there are no more {@apilink Request} objects to crawl. * - * We can use the `preNavigationHooks` to adjust `gotOptions`: + * We can use the `preNavigationHooks` to adjust the crawling context before the request is made: * * ```javascript * preNavigationHooks: [ - * (crawlingContext, gotOptions) => { + * (crawlingContext) => { * // ... * }, * ] @@ -328,17 +311,12 @@ export type HttpRequestHandler< * @category Crawlers */ export class HttpCrawler< - Context extends InternalHttpCrawlingContext>, -> extends BasicCrawler { - /** - * A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies. - * Only available if used by the crawler. - */ - proxyConfiguration?: ProxyConfiguration; - - protected userRequestHandlerTimeoutMillis: number; - protected preNavigationHooks: InternalHttpHook[]; - protected postNavigationHooks: InternalHttpHook[]; + Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, + ContextExtension = Dictionary, + ExtendedContext extends Context = Context & ContextExtension, +> extends BasicCrawler { + protected preNavigationHooks: InternalHttpHook[]; + protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable)[]; protected persistCookiesPerSession: boolean; protected navigationTimeoutMillis: number; protected ignoreSslErrors: boolean; @@ -350,14 +328,12 @@ export class HttpCrawler< protected static override optionsShape = { ...BasicCrawler.optionsShape, - handlePageFunction: ow.optional.function, navigationTimeoutSecs: ow.optional.number, ignoreSslErrors: ow.optional.boolean, additionalMimeTypes: ow.optional.array.ofType(ow.string), suggestResponseEncoding: ow.optional.string, forceResponseEncoding: ow.optional.string, - proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration), persistCookiesPerSession: ow.optional.boolean, additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number), @@ -371,62 +347,41 @@ export class HttpCrawler< * All `HttpCrawlerOptions` parameters are passed via an options object. */ constructor( - options: HttpCrawlerOptions = {}, + options: HttpCrawlerOptions & + RequireContextPipeline = {} as any, override readonly config = Configuration.getGlobalConfig(), ) { ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape)); const { - requestHandler, - handlePageFunction, - - requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, - proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], - // Ignored - handleRequestFunction, - // BasicCrawler autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, + contextPipelineBuilder, ...basicCrawlerOptions } = options; super( { ...basicCrawlerOptions, - requestHandler, autoscaledPoolOptions, - // We need to add some time for internal functions to finish, - // but not too much so that we would stall the crawler. - requestHandlerTimeoutSecs: - navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, + contextPipelineBuilder: + contextPipelineBuilder ?? + (() => this.buildContextPipeline() as ContextPipeline), }, config, ); - this._handlePropertyNameChange({ - newName: 'requestHandler', - oldName: 'handlePageFunction', - propertyKey: 'requestHandler', - newProperty: requestHandler, - oldProperty: handlePageFunction, - allowUndefined: true, - }); - - if (!this.requestHandler) { - this.requestHandler = this.router; - } - // Cookies should be persisted per session only if session pool is used if (!this.useSessionPool && persistCookiesPerSession) { throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.'); @@ -441,14 +396,12 @@ export class HttpCrawler< ); } - this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000; this.navigationTimeoutMillis = navigationTimeoutSecs * 1000; this.ignoreSslErrors = ignoreSslErrors; this.suggestResponseEncoding = suggestResponseEncoding; this.forceResponseEncoding = forceResponseEncoding; this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]); this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]); - this.proxyConfiguration = proxyConfiguration; this.preNavigationHooks = preNavigationHooks; this.postNavigationHooks = [ ({ request, response }) => this._abortDownloadOfBody(request, response!), @@ -462,138 +415,145 @@ export class HttpCrawler< } } - /** - * **EXPERIMENTAL** - * Function for attaching CrawlerExtensions such as the Unblockers. - * @param extension Crawler extension that overrides the crawler configuration. - */ - use(extension: CrawlerExtension) { - ow(extension, ow.object.instanceOf(CrawlerExtension)); - - const className = this.constructor.name; - - const extensionOptions = extension.getCrawlerOptions(); - - for (const [key, value] of Object.entries(extensionOptions)) { - const isConfigurable = Object.hasOwn(this, key); - const originalType = typeof this[key as keyof this]; - const extensionType = typeof value; // What if we want to null something? It is really needed? - const isSameType = originalType === extensionType || value == null; // fast track for deleting keys - const exists = this[key as keyof this] != null; - - if (!isConfigurable) { - // Test if the property can be configured on the crawler - throw new Error( - `${extension.name} tries to set property "${key}" that is not configurable on ${className} instance.`, - ); - } - - if (!isSameType && exists) { - // Assuming that extensions will only add up configuration - throw new Error( - `${extension.name} tries to set property of different type "${extensionType}". "${className}.${key}: ${originalType}".`, - ); - } - - this.log.warning(`${extension.name} is overriding "${className}.${key}: ${originalType}" with ${value}.`); - - this[key as keyof this] = value as this[keyof this]; - } + protected buildContextPipeline(): ContextPipeline { + return ContextPipeline.create() + .compose({ + action: this.makeHttpRequest.bind(this), + }) + .compose({ action: this.processHttpResponse.bind(this) }) + .compose({ action: this.handleBlockedRequestByContent.bind(this) }); } - /** - * Wrapper around requestHandler that opens and closes pages etc. - */ - protected override async _runRequestHandler(crawlingContext: Context) { + private async makeHttpRequest( + crawlingContext: CrawlingContext, + ): Promise & Partial> { const { request, session } = crawlingContext; - if (this.proxyConfiguration) { - const sessionId = session ? session.id : undefined; - crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request }); + if (request.skipNavigation) { + return { + request: new Proxy(request, { + get(target, propertyName, receiver) { + if (propertyName === 'loadedUrl') { + throw new Error( + 'The `request.loadedUrl` property is not available - `skipNavigation` was used', + ); + } + return Reflect.get(target, propertyName, receiver); + }, + }) as LoadedRequest, + get response(): InternalHttpCrawlingContext['response'] { + throw new Error('The `response` property is not available - `skipNavigation` was used'); + }, + }; } - if (!request.skipNavigation) { - await this._handleNavigation(crawlingContext); - tryCancel(); + const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request); - const parsed = await this._parseResponse(request, crawlingContext.response!, crawlingContext); - const response = parsed.response!; - const contentType = parsed.contentType!; - tryCancel(); + request.state = RequestState.BEFORE_NAV; + // Execute pre navigation hooks before applying session pool cookies, + // as they may also set cookies in the session + await this._executeHooks(this.preNavigationHooks, crawlingContext); + tryCancel(); - // `??=` because descendant classes may already set optimized version - crawlingContext.waitForSelector ??= async (selector?: string, _timeoutMs?: number) => { - const $ = cheerio.load(parsed.body!.toString()); + const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request); - if ($(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } - }; - crawlingContext.parseWithCheerio ??= async (selector?: string, timeoutMs?: number) => { - const $ = cheerio.load(parsed.body!.toString()); + const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies); - if (selector) { - await crawlingContext.waitForSelector(selector, timeoutMs); - } + const proxyUrl = crawlingContext.proxyInfo?.url; - return $; - }; + const httpResponse = await addTimeoutToPromise( + async () => this._requestFunction({ request, session, proxyUrl, cookieString }), + this.navigationTimeoutMillis, + `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`, + ); + tryCancel(); - if (this.useSessionPool) { - this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!); - } + request.loadedUrl = httpResponse?.url; + request.state = RequestState.AFTER_NAV; - if (this.persistCookiesPerSession) { - crawlingContext.session!.setCookiesFromResponse(response); - } + return { request: request as LoadedRequest, response: httpResponse }; + } + + private async processHttpResponse( + crawlingContext: CrawlingContextWithReponse, + ): Promise< + Omit & Partial + > { + if (crawlingContext.request.skipNavigation) { + return { + get contentType(): InternalHttpCrawlingContext['contentType'] { + throw new Error('The `contentType` property is not available - `skipNavigation` was used'); + }, + get body(): InternalHttpCrawlingContext['body'] { + throw new Error('The `body` property is not available - `skipNavigation` was used'); + }, + get json(): InternalHttpCrawlingContext['json'] { + throw new Error('The `json` property is not available - `skipNavigation` was used'); + }, + get waitForSelector(): InternalHttpCrawlingContext['waitForSelector'] { + throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used'); + }, + get parseWithCheerio(): InternalHttpCrawlingContext['parseWithCheerio'] { + throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used'); + }, + }; + } - request.loadedUrl = response.url; + await this._executeHooks(this.postNavigationHooks, crawlingContext); + tryCancel(); - if (!this.requestMatchesEnqueueStrategy(request)) { - this.log.debug( - // eslint-disable-next-line dot-notation - `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`, - ); + const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response); + tryCancel(); + const response = parsed.response!; + const contentType = parsed.contentType!; - request.noRetry = true; - request.state = RequestState.SKIPPED; + const waitForSelector = async (selector: string, _timeoutMs?: number) => { + const $ = cheerio.load(parsed.body!.toString()); - await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); + if ($(selector).get().length === 0) { + throw new Error(`Selector '${selector}' not found.`); + } + }; + const parseWithCheerio = async (selector?: string, timeoutMs?: number) => { + const $ = cheerio.load(parsed.body!.toString()); - return; + if (selector) { + await (crawlingContext as InternalHttpCrawlingContext).waitForSelector(selector, timeoutMs); } - Object.assign(crawlingContext, parsed); + return $; + }; + + if (this.useSessionPool) { + this._throwOnBlockedRequest(crawlingContext.session!, response.status!); + } - Object.defineProperty(crawlingContext, 'json', { - get() { - if (contentType.type !== APPLICATION_JSON_MIME_TYPE) return null; - const jsonString = parsed.body!.toString(contentType.encoding); - return JSON.parse(jsonString); - }, - }); + if (this.persistCookiesPerSession) { + crawlingContext.session!.setCookiesFromResponse(response); } + return { + get json() { + if (contentType.type !== APPLICATION_JSON_MIME_TYPE) return null; + const jsonString = parsed.body!.toString(contentType.encoding); + return JSON.parse(jsonString); + }, + waitForSelector, + parseWithCheerio, + contentType, + body: parsed.body, + }; + } + + private async handleBlockedRequestByContent(crawlingContext: InternalHttpCrawlingContext): Promise<{}> { if (this.retryOnBlocked) { const error = await this.isRequestBlocked(crawlingContext); if (error) throw new SessionError(error); } - - request.state = RequestState.REQUEST_HANDLER; - try { - await addTimeoutToPromise( - async () => Promise.resolve(this.requestHandler(crawlingContext as LoadedContext)), - this.userRequestHandlerTimeoutMillis, - `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`, - ); - request.state = RequestState.DONE; - } catch (e: any) { - request.state = RequestState.ERROR; - throw e; - } + return {}; } - protected override async isRequestBlocked(crawlingContext: Context): Promise { + protected async isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise { if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) { const $ = await crawlingContext.parseWithCheerio(); @@ -611,97 +571,27 @@ export class HttpCrawler< this.sessionPool!['blockedStatusCodes'] : BLOCKED_STATUS_CODES; - if (blockedStatusCodes.includes(crawlingContext.response.statusCode!)) { - return `Blocked by status code ${crawlingContext.response.statusCode}`; + if (blockedStatusCodes.includes(crawlingContext.response.status!)) { + return `Blocked by status code ${crawlingContext.response.status}`; } return false; } - protected async _handleNavigation(crawlingContext: Context) { - const gotOptions = {} as OptionsInit; - const { request, session } = crawlingContext; - const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request); - - request.state = RequestState.BEFORE_NAV; - // Execute pre navigation hooks before applying session pool cookies, - // as they may also set cookies in the session - await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions); - tryCancel(); - - const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request); - - this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies); - - const proxyUrl = crawlingContext.proxyInfo?.url; - - crawlingContext.response = await addTimeoutToPromise( - async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), - this.navigationTimeoutMillis, - `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`, - ); - tryCancel(); - - request.state = RequestState.AFTER_NAV; - await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions); - tryCancel(); - } - /** - * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks. + * Returns the `Cookie` header value based on the current context and + * any changes that occurred in the navigation hooks. */ protected _applyCookies( { session, request }: CrawlingContext, - gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string, - ) { + ): string { const sessionCookie = session?.getCookieString(request.url) ?? ''; - let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || ''; - - if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) { - const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers; - - this.log.warning( - `Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`, - ); - - const sourceCookies = []; - - if (Array.isArray(lowerCaseHeader)) { - sourceCookies.push(...lowerCaseHeader); - } else { - sourceCookies.push(lowerCaseHeader); - } - - if (Array.isArray(upperCaseHeader)) { - sourceCookies.push(...upperCaseHeader); - } else { - sourceCookies.push(upperCaseHeader); - } - - alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies); - } - const sourceCookies = [sessionCookie, preHookCookies]; - - if (Array.isArray(alteredGotOptionsCookies)) { - sourceCookies.push(...alteredGotOptionsCookies); - } else { - sourceCookies.push(alteredGotOptionsCookies); - } + const sourceCookies = [sessionCookie, preHookCookies, postHookCookies]; - sourceCookies.push(postHookCookies); - - const mergedCookie = mergeCookies(request.url, sourceCookies); - - gotOptions.headers ??= {}; - Reflect.deleteProperty(gotOptions.headers, 'Cookie'); - Reflect.deleteProperty(gotOptions.headers, 'cookie'); - - if (mergedCookie !== '') { - gotOptions.headers.Cookie = mergedCookie; - } + return mergeCookies(request.url, sourceCookies); } /** @@ -713,21 +603,16 @@ export class HttpCrawler< request, session, proxyUrl, - gotOptions, - }: RequestFunctionOptions): Promise { - if (!TimeoutError) { - // @ts-ignore - ({ TimeoutError } = await import('got-scraping')); - } - - const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions); + cookieString, + }: RequestFunctionOptions): Promise { + const opts = this._getRequestOptions(request, session, proxyUrl); try { - return await this._requestAsBrowser(opts, session); + return await this._requestAsBrowser(opts, session, cookieString); } catch (e) { - if (e instanceof TimeoutError) { + if (e instanceof Error && e.constructor.name === 'TimeoutError') { this._handleRequestTimeout(session); - return undefined as unknown as PlainResponse; + return new Response(); // this will never happen, as _handleRequestTimeout always throws } if (this.isProxyError(e as Error)) { @@ -741,21 +626,21 @@ export class HttpCrawler< /** * Encodes and parses response according to the provided content type */ - protected async _parseResponse(request: Request, responseStream: IncomingMessage, crawlingContext: Context) { - const { statusCode } = responseStream; - const { type, charset } = parseContentTypeFromResponse(responseStream); - const { response, encoding } = this._encodeResponse(request, responseStream, charset); + protected async _parseResponse(request: CrawleeRequest, response: Response) { + const { status } = response; + const { type, charset } = parseContentTypeFromResponse(response); + const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset); const contentType = { type, encoding }; - if (statusCode! >= 400 && statusCode! <= 599) { - this.stats.registerStatusCode(statusCode!); + if (status >= 400 && status <= 599) { + this.stats.registerStatusCode(status); } - const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode!); - const includeError = this.additionalHttpErrorStatusCodes.has(statusCode!); + const excludeError = this.ignoreHttpErrorStatusCodes.has(status); + const includeError = this.additionalHttpErrorStatusCodes.has(status); - if ((statusCode! >= 500 && !excludeError) || includeError) { - const body = await readStreamToString(response, encoding); + if ((status >= 500 && !excludeError) || includeError) { + const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text) // Errors are often sent as JSON, so attempt to parse them, // despite Accept header being set to text/html. @@ -763,68 +648,50 @@ export class HttpCrawler< const errorResponse = JSON.parse(body); let { message } = errorResponse; if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 }); - throw new Error(`${statusCode} - ${message}`); + throw new Error(`${status} - ${message}`); } if (includeError) { - throw new Error(`${statusCode} - Error status code was set by user.`); + throw new Error(`${status} - Error status code was set by user.`); } // It's not a JSON, so it's probably some text. Get the first 100 chars of it. - throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`); + throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`); } else if (HTML_AND_XML_MIME_TYPES.includes(type)) { - const isXml = type.includes('xml'); - const parsed = await this._parseHTML(response, isXml, crawlingContext); - return { ...parsed, isXml, response, contentType }; + return { response, contentType, body: await reencodedResponse.text() }; } else { - const body = await concatStreamToBuffer(response); + const body = Buffer.from(await reencodedResponse.bytes()); return { body, response, contentType, - enqueueLinks: async () => Promise.resolve({ processedRequests: [], unprocessedRequests: [] }), }; } } - protected async _parseHTML( - response: IncomingMessage, - _isXml: boolean, - _crawlingContext: Context, - ): Promise> { - return { - body: await concatStreamToBuffer(response), - } as Partial; - } - /** * Combines the provided `requestOptions` with mandatory (non-overridable) values. */ - protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit) { - const requestOptions: OptionsInit & Required> & { isStream: true } = { + protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string) { + const requestOptions = { url: request.url, - method: request.method as Method, + method: request.method, proxyUrl, - timeout: { request: this.navigationTimeoutMillis }, + timeout: this.navigationTimeoutMillis, cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined, sessionToken: session, - ...gotOptions, - headers: { ...request.headers, ...gotOptions?.headers }, + headers: request.headers, https: { - ...gotOptions?.https, rejectUnauthorized: !this.ignoreSslErrors, }, - isStream: true, + body: undefined as string | undefined, }; // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header Reflect.deleteProperty(requestOptions.headers!, 'cookie'); - // TODO this is incorrect, the check for man in the middle needs to be done - // on individual proxy level, not on the `proxyConfiguration` level, - // because users can use normal + MITM proxies in a single configuration. // Disable SSL verification for MITM proxies - if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) { + if (session?.proxyInfo?.ignoreTlsErrors) { requestOptions.https = { ...requestOptions.https, rejectUnauthorized: false, @@ -837,12 +704,12 @@ export class HttpCrawler< } protected _encodeResponse( - request: Request, - response: IncomingMessage, + request: CrawleeRequest, + response: Response, encoding: BufferEncoding, ): { encoding: BufferEncoding; - response: IncomingMessage; + response: Response; } { if (this.forceResponseEncoding) { encoding = this.forceResponseEncoding as BufferEncoding; @@ -862,17 +729,18 @@ export class HttpCrawler< if (iconv.encodingExists(encoding)) { const encodeStream = iconv.encodeStream(utf8); const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err)); - response.on('error', (err: Error) => decodeStream.emit('error', err)); - const encodedResponse = response.pipe(decodeStream).pipe(encodeStream) as NodeJS.ReadWriteStream & { - statusCode?: number; - headers: IncomingHttpHeaders; - url?: string; - }; - encodedResponse.statusCode = response.statusCode; - encodedResponse.headers = response.headers; - encodedResponse.url = response.url; + const reencodedBody = response.body + ? Readable.toWeb( + Readable.from( + Readable.fromWeb(response.body as any) + .pipe(decodeStream) + .pipe(encodeStream), + ), + ) + : null; + return { - response: encodedResponse as any, + response: new ResponseWithUrl(reencodedBody as any, response), encoding: utf8, }; } @@ -904,17 +772,17 @@ export class HttpCrawler< */ protected _handleRequestTimeout(session?: Session) { session?.markBad(); - throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`); + throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`); } - private _abortDownloadOfBody(request: Request, response: IncomingMessage) { - const { statusCode } = response; + private _abortDownloadOfBody(request: CrawleeRequest, response: Response) { + const { status } = response; const { type } = parseContentTypeFromResponse(response); // eslint-disable-next-line dot-notation -- accessing private property const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : []; // if we retry the request, can the Content-Type change? - const isTransientContentType = statusCode! >= 500 || blockedStatusCodes.includes(statusCode!); + const isTransientContentType = status >= 500 || blockedStatusCodes.includes(status); if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) { request.noRetry = true; @@ -928,117 +796,42 @@ export class HttpCrawler< /** * @internal wraps public utility for mocking purposes */ - private _requestAsBrowser = async ( - options: OptionsInit & { url: string | URL; isStream: true }, - session?: Session, - ) => { - const response = await this.httpClient.stream( - processHttpRequestOptions({ - ...(options as any), - cookieJar: options.cookieJar, - responseType: 'text', - }), - (redirectResponse, updatedRequest) => { - if (this.persistCookiesPerSession) { - session!.setCookiesFromResponse(redirectResponse); - - const cookieString = session!.getCookieString(updatedRequest.url!.toString()); - if (cookieString !== '') { - updatedRequest.headers.Cookie = cookieString; - } - } + private _requestAsBrowser = async (options: Dictionary, session?: Session, cookieString?: string) => { + const opts = processHttpRequestOptions({ + ...(options as any), + cookieJar: options.cookieJar, + responseType: 'text', + }); + + if (cookieString) { + opts.headers?.delete('Cookie'); + opts.headers?.delete('cookie'); + opts.headers?.set('Cookie', cookieString); + } + + const response = await this.httpClient.sendRequest( + new Request(opts.url, { + body: opts.body ? (Readable.toWeb(opts.body) as any) : undefined, + headers: new Headers(opts.headers), + method: opts.method, + // Node-specific option to make the request body work with streams + duplex: 'half', + } as RequestInit), + { + session, + timeout: opts.timeout, }, ); - return addResponsePropertiesToStream(response.stream, response); + return response; }; } interface RequestFunctionOptions { - request: Request; + request: CrawleeRequest; session?: Session; proxyUrl?: string; - gotOptions: OptionsInit; -} - -/** - * The stream object returned from got does not have the below properties. - * At the same time, you can't read data directly from the response stream, - * because they won't get emitted unless you also read from the primary - * got stream. To be able to work with only one stream, we move the expected props - * from the response stream to the got stream. - * @internal - */ -function addResponsePropertiesToStream(stream: Readable, response: StreamingHttpResponse) { - const properties: (keyof PlainResponse)[] = [ - 'statusCode', - 'statusMessage', - 'headers', - 'complete', - 'httpVersion', - 'rawHeaders', - 'rawTrailers', - 'trailers', - 'url', - 'request', - ]; - - stream.on('end', () => { - // @ts-expect-error - if (stream.rawTrailers) stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0 - - // @ts-expect-error - if (stream.trailers) stream.trailers = response.trailers; - - // @ts-expect-error - stream.complete = response.complete; - }); - - for (const prop of properties) { - if (!(prop in stream)) { - (stream as any)[prop] = (response as any)[prop]; - } - } - - return stream as unknown as PlainResponse; -} - -/** - * Gets parsed content type from response object - * @param response HTTP response object - */ -function parseContentTypeFromResponse(response: unknown): { type: string; charset: BufferEncoding } { - ow( - response, - ow.object.partialShape({ - url: ow.string.url, - headers: new ObjectPredicate>(), - }), - ); - - const { url, headers } = response; - let parsedContentType; - - if (headers['content-type']) { - try { - parsedContentType = contentTypeParser.parse(headers['content-type'] as string); - } catch { - // Can not parse content type from Content-Type header. Try to parse it from file extension. - } - } - - // Parse content type from file extension as fallback - if (!parsedContentType) { - const parsedUrl = new URL(url); - const contentTypeFromExtname = - mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5 - parsedContentType = contentTypeParser.parse(contentTypeFromExtname); - } - - return { - type: parsedContentType.type, - charset: parsedContentType.parameters.charset as BufferEncoding, - }; + cookieString?: string; } /** diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.ts new file mode 100644 index 000000000000..4111da0ea169 --- /dev/null +++ b/packages/http-crawler/src/internals/utils.ts @@ -0,0 +1,98 @@ +import { extname } from 'node:path'; +import { Readable } from 'node:stream'; + +import type { HttpRequest, HttpRequestOptions } from '@crawlee/types'; +import { applySearchParams } from '@crawlee/utils'; +import contentTypeParser from 'content-type'; +import mime from 'mime-types'; +import ow, { ObjectPredicate } from 'ow'; + +/** + * Converts {@apilink HttpRequestOptions} to a {@apilink HttpRequest}. + */ +export function processHttpRequestOptions({ + searchParams, + form, + json, + username, + password, + ...request +}: HttpRequestOptions): HttpRequest { + const url = new URL(request.url); + const headers = new Headers(request.headers); + + applySearchParams(url, searchParams); + + if ([request.body, form, json].filter((value) => value !== undefined).length > 1) { + throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments'); + } + + const body = (() => { + if (form !== undefined) { + return Readable.from(new URLSearchParams(form).toString()); + } + + if (json !== undefined) { + return Readable.from(JSON.stringify(json)); + } + + if (request.body !== undefined) { + return Readable.from(request.body); + } + + return undefined; + })(); + + if (form !== undefined && !headers.has('content-type')) { + headers.set('content-type', 'application/x-www-form-urlencoded'); + } + + if (json !== undefined && !headers.has('content-type')) { + headers.set('content-type', 'application/json'); + } + + if (username !== undefined || password !== undefined) { + const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64'); + headers.set('authorization', `Basic ${encodedAuth}`); + } + + return { ...request, body, url, headers }; +} + +/** + * Gets parsed content type from response object + * @param response HTTP response object + */ +export function parseContentTypeFromResponse(response: Response): { type: string; charset: BufferEncoding } { + ow( + response, + ow.object.partialShape({ + url: ow.string.url, + headers: new ObjectPredicate>(), + }), + ); + + const { url, headers } = response; + let parsedContentType; + + if (headers.get('content-type')) { + try { + parsedContentType = contentTypeParser.parse(headers.get('content-type') as string); + } catch { + // Can not parse content type from Content-Type header. Try to parse it from file extension. + } + } + + // Parse content type from file extension as fallback + if (!parsedContentType) { + const parsedUrl = new URL(url); + const contentTypeFromExtname = + mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5 + parsedContentType = contentTypeParser.parse(contentTypeFromExtname); + } + + return { + type: parsedContentType.type, + charset: parsedContentType.parameters.charset as BufferEncoding, + }; +} diff --git a/packages/impit-client/package.json b/packages/impit-client/package.json index b6e2cd783467..c47fd75395e1 100644 --- a/packages/impit-client/package.json +++ b/packages/impit-client/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/impit-client", - "version": "3.15.3", + "version": "4.0.0", "description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.", "engines": { - "node": ">=20.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,21 +40,16 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, - "peerDependencies": { - "@crawlee/core": "^3.12.1" - }, - "devDependencies": { - "@crawlee/core": "^3.15.3" - }, "dependencies": { "@apify/datastructures": "^2.0.3", - "impit": "^0.7.0", + "@crawlee/http-client": "4.0.0", + "impit": "^0.8.2", "tough-cookie": "^6.0.0" } } diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.ts index 94e4c3052e19..03286d5f1b50 100644 --- a/packages/impit-client/src/index.ts +++ b/packages/impit-client/src/index.ts @@ -1,10 +1,6 @@ -import { Readable } from 'node:stream'; -import { type ReadableStream } from 'node:stream/web'; -import { isGeneratorObject } from 'node:util/types'; - -import type { BaseHttpClient, HttpRequest, HttpResponse, ResponseTypes, StreamingHttpResponse } from '@crawlee/core'; -import type { HttpMethod, ImpitOptions, ImpitResponse, RequestInit } from 'impit'; -import { Impit } from 'impit'; +import type { CustomFetchOptions } from '@crawlee/http-client'; +import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client'; +import { Impit, type ImpitOptions } from 'impit'; import type { CookieJar as ToughCookieJar } from 'tough-cookie'; import { LruCache } from '@apify/datastructures'; @@ -14,18 +10,11 @@ export const Browser = { 'Firefox': 'firefox', } as const; -interface ResponseWithRedirects { - response: ImpitResponse; - redirectUrls: URL[]; -} - /** - * A HTTP client implementation based on the `impit library. + * A HTTP client implementation based on the `impit` library. */ -export class ImpitHttpClient implements BaseHttpClient { +export class ImpitHttpClient extends BaseHttpClient { private impitOptions: ImpitOptions; - private maxRedirects: number; - private followRedirects: boolean; /** * Enables reuse of `impit` clients for the same set of options. @@ -51,188 +40,26 @@ export class ImpitHttpClient implements BaseHttpClient { return client; } - constructor(options?: Omit & { maxRedirects?: number }) { + constructor(options?: Omit) { + super(); this.impitOptions = options ?? {}; - - this.maxRedirects = options?.maxRedirects ?? 10; - this.followRedirects = options?.followRedirects ?? true; - } - - /** - * Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`. - * @param headers `SimpleHeaders` object - * @returns `Record` object - */ - private intoHeaders( - headers?: Exclude['headers'], undefined>, - ): Headers | undefined { - if (!headers) { - return undefined; - } - - const result = new Headers(); - - for (const headerName of Object.keys(headers)) { - const headerValue = headers[headerName]; - - for (const value of Array.isArray(headerValue) ? headerValue : [headerValue]) { - if (value === undefined) continue; - - result.append(headerName, value); - } - } - - return result; - } - - private intoImpitBody( - body?: Exclude['body'], undefined>, - ): RequestInit['body'] { - if (isGeneratorObject(body)) { - return Readable.toWeb(Readable.from(body)) as any; - } - if (body instanceof Readable) { - return Readable.toWeb(body) as any; - } - - return body as any; } /** - * Common implementation for `sendRequest` and `stream` methods. - * @param request `HttpRequest` object - * @returns `HttpResponse` object + * @inheritDoc */ - private async getResponse( - request: HttpRequest, - redirects?: { - redirectCount?: number; - redirectUrls?: URL[]; - }, - ): Promise { - if ((redirects?.redirectCount ?? 0) > this.maxRedirects) { - throw new Error(`Too many redirects, maximum is ${this.maxRedirects}.`); - } - - const url = typeof request.url === 'string' ? request.url : request.url.href; + async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise { + const { proxyUrl, redirect, signal } = options ?? {}; const impit = this.getClient({ ...this.impitOptions, - ...(request?.cookieJar ? { cookieJar: request.cookieJar as ToughCookieJar } : {}), - proxyUrl: request.proxyUrl, - followRedirects: false, - }); - - const response = await impit.fetch(url, { - method: request.method as HttpMethod, - headers: this.intoHeaders(request.headers), - body: this.intoImpitBody(request.body), - timeout: (request.timeout as { request?: number })?.request, - }); - - if (this.followRedirects && response.status >= 300 && response.status < 400) { - const location = response.headers.get('location'); - const redirectUrl = new URL(location ?? '', request.url); - - if (!location) { - throw new Error('Redirect response missing location header.'); - } - - return this.getResponse( - { - ...request, - url: redirectUrl.href, - }, - { - redirectCount: (redirects?.redirectCount ?? 0) + 1, - redirectUrls: [...(redirects?.redirectUrls ?? []), redirectUrl], - }, - ); - } - - return { - response, - redirectUrls: redirects?.redirectUrls ?? [], - }; - } - - /** - * @inheritDoc - */ - async sendRequest( - request: HttpRequest, - ): Promise> { - const { response, redirectUrls } = await this.getResponse(request); - - let responseBody; - - switch (request.responseType) { - case 'text': - responseBody = await response.text(); - break; - case 'json': - responseBody = await response.json(); - break; - case 'buffer': - responseBody = await response.bytes(); - break; - default: - throw new Error('Unsupported response type.'); - } - - return { - headers: Object.fromEntries(response.headers.entries()), - statusCode: response.status, - url: response.url, - request, - redirectUrls, - trailers: {}, - body: responseBody, - complete: true, - }; - } - - private getStreamWithProgress( - response: ImpitResponse, - ): [Readable, () => { percent: number; transferred: number; total: number }] { - const responseStream = Readable.fromWeb(response.body as ReadableStream); - let transferred = 0; - const total = Number(response.headers.get('content-length') ?? 0); - responseStream.on('data', (chunk) => { - transferred += chunk.length; + proxyUrl, + followRedirects: redirect === 'follow', }); - const getDownloadProgress = () => { - return { - percent: Math.round((transferred / total) * 100), - transferred, - total, - }; - }; - - return [responseStream, getDownloadProgress]; - } - - /** - * @inheritDoc - */ - async stream(request: HttpRequest): Promise { - const { response, redirectUrls } = await this.getResponse(request); - const [stream, getDownloadProgress] = this.getStreamWithProgress(response); + const response = await impit.fetch(request, { signal: signal ?? undefined }); - return { - request, - url: response.url, - statusCode: response.status, - stream, - complete: true, - get downloadProgress() { - return getDownloadProgress(); - }, - uploadProgress: { percent: 100, transferred: 0 }, - redirectUrls, - headers: Object.fromEntries(response.headers.entries()), - trailers: {}, - }; + // todo - cast shouldn't be needed here, impit returns `Uint8Array` + return new ResponseWithUrl(response.body, response); } } diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json index be624fdb2529..d891d25d75ee 100644 --- a/packages/jsdom-crawler/package.json +++ b/packages/jsdom-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/jsdom", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,7 +40,7 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { @@ -55,13 +49,13 @@ "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "@types/jsdom": "^21.0.0", - "cheerio": "1.0.0-rc.12", - "jsdom": "^26.0.0", - "ow": "^0.28.2", - "tslib": "^2.4.0" + "@crawlee/http": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "@types/jsdom": "^21.1.7", + "cheerio": "^1.0.0", + "jsdom": "^26.1.0", + "ow": "^2.0.0", + "tslib": "^2.8.1" } } diff --git a/packages/jsdom-crawler/src/index.ts b/packages/jsdom-crawler/src/index.ts index 2a7454461457..905025dc8d63 100644 --- a/packages/jsdom-crawler/src/index.ts +++ b/packages/jsdom-crawler/src/index.ts @@ -1,2 +1,2 @@ export * from '@crawlee/http'; -export * from './internals/jsdom-crawler'; +export * from './internals/jsdom-crawler.js'; diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts index 0178684af312..75214fd2528b 100644 --- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts +++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts @@ -1,5 +1,3 @@ -import type { IncomingMessage } from 'node:http'; - import type { BasicCrawlingContext, Configuration, @@ -29,7 +27,6 @@ import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom'; import ow from 'ow'; import { addTimeoutToPromise } from '@apify/timeout'; -import { concatStreamToBuffer } from '@apify/utilities'; export type JSDOMErrorHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler @@ -37,9 +34,11 @@ export type JSDOMErrorHandler< > = ErrorHandler>; export interface JSDOMCrawlerOptions< + ContextExtension = Dictionary, + ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends HttpCrawlerOptions> { +> extends HttpCrawlerOptions, ContextExtension, ExtendedContext> { /** * Download and run scripts. */ @@ -58,10 +57,12 @@ export type JSDOMHook< export interface JSDOMCrawlingContext< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends InternalHttpCrawlingContext { +> extends InternalHttpCrawlingContext { window: DOMWindow; document: Document; + body: string; + /** * Wait for an element matching the selector to appear. * Timeout defaults to 5s. @@ -129,11 +130,11 @@ export type JSDOMRequestHandler< * * The crawler finishes when there are no more {@apilink Request} objects to crawl. * - * We can use the `preNavigationHooks` to adjust `gotOptions`: + * We can use the `preNavigationHooks` to adjust the crawling context before the request is made: * * ``` * preNavigationHooks: [ - * (crawlingContext, gotOptions) => { + * (crawlingContext) => { * // ... * }, * ] @@ -177,7 +178,10 @@ const resources = new ResourceLoader({ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', }); -export class JSDOMCrawler extends HttpCrawler { +export class JSDOMCrawler< + ContextExtension = Dictionary, + ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, +> extends HttpCrawler { protected static override optionsShape = { ...HttpCrawler.optionsShape, runScripts: ow.optional.boolean, @@ -188,10 +192,25 @@ export class JSDOMCrawler extends HttpCrawler { protected hideInternalConsole: boolean; protected virtualConsole: VirtualConsole | null = null; - constructor(options: JSDOMCrawlerOptions = {}, config?: Configuration) { + constructor(options: JSDOMCrawlerOptions = {}, config?: Configuration) { const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options; - super(httpOptions, config); + super( + { + ...httpOptions, + contextPipelineBuilder: () => + this.buildContextPipeline() + .compose({ + action: async (context) => await this.parseContent(context), + cleanup: async (context) => { + this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler); + context.window?.close(); + }, + }) + .compose({ action: async (context) => await this.addHelpers(context) }), + }, + config, + ); this.runScripts = runScripts; this.hideInternalConsole = hideInternalConsole; @@ -229,20 +248,12 @@ export class JSDOMCrawler extends HttpCrawler { private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', error); - protected override async _cleanupContext(context: JSDOMCrawlingContext) { - this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler); - context.window?.close(); - } - - protected override async _parseHTML( - response: IncomingMessage, - isXml: boolean, - crawlingContext: JSDOMCrawlingContext, - ) { - const body = await concatStreamToBuffer(response); + private async parseContent(crawlingContext: InternalHttpCrawlingContext) { + const isXml = crawlingContext.contentType.type.includes('xml'); - const { window } = new JSDOM(body, { - url: response.url, + // TODO handle non-string + const { window } = new JSDOM(crawlingContext.body.toString(), { + url: crawlingContext.response.url, contentType: isXml ? 'text/xml' : 'text/html', runScripts: this.runScripts ? 'dangerously' : undefined, resources, @@ -301,10 +312,15 @@ export class JSDOMCrawler extends HttpCrawler { get document() { return window.document; }, + }; + } + + private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string; window: DOMWindow }) { + return { enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => { return domCrawlerEnqueueLinks({ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) }, - window, + window: crawlingContext.window, requestQueue: await this.getRequestQueue(), robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url), onSkippedRequest: this.handleSkippedRequest, @@ -312,34 +328,29 @@ export class JSDOMCrawler extends HttpCrawler { finalRequestUrl: crawlingContext.request.loadedUrl, }); }, - }; - } + async waitForSelector(selector: string, timeoutMs = 5_000) { + const $ = cheerio.load(crawlingContext.body); - override async _runRequestHandler(context: JSDOMCrawlingContext) { - context.waitForSelector = async (selector: string, timeoutMs = 5_000) => { - const $ = cheerio.load(context.body); + if ($(selector).get().length === 0) { + if (timeoutMs) { + await sleep(50); + await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0)); + return; + } - if ($(selector).get().length === 0) { - if (timeoutMs) { - await sleep(50); - await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0)); - return; + throw new Error(`Selector '${selector}' not found.`); } + }, + async parseWithCheerio(selector?: string, _timeoutMs = 5_000) { + const $ = cheerio.load(crawlingContext.body); - throw new Error(`Selector '${selector}' not found.`); - } - }; - context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => { - const $ = cheerio.load(context.body); - - if (selector && $(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } + if (selector && $(selector).get().length === 0) { + throw new Error(`Selector '${selector}' not found.`); + } - return $; + return $; + }, }; - - await super._runRequestHandler(context); } } diff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.json index 6d0f167b4663..2b16f759974c 100644 --- a/packages/linkedom-crawler/package.json +++ b/packages/linkedom-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/linkedom", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,19 +40,19 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.15.3", - "@crawlee/types": "3.15.3", - "linkedom": "^0.18.0", - "ow": "^0.28.2", - "tslib": "^2.4.0" + "@apify/timeout": "^0.3.2", + "@apify/utilities": "^2.15.5", + "@crawlee/http": "4.0.0", + "@crawlee/types": "4.0.0", + "linkedom": "^0.18.10", + "ow": "^2.0.0", + "tslib": "^2.8.1" } } diff --git a/packages/linkedom-crawler/src/index.ts b/packages/linkedom-crawler/src/index.ts index c52d14dcb12a..ab8cc478d1c7 100644 --- a/packages/linkedom-crawler/src/index.ts +++ b/packages/linkedom-crawler/src/index.ts @@ -1,2 +1,2 @@ export * from '@crawlee/http'; -export * from './internals/linkedom-crawler'; +export * from './internals/linkedom-crawler.js'; diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts index bb06f6a2fc7d..169a03efe8b2 100644 --- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts +++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts @@ -1,5 +1,3 @@ -import type { IncomingMessage } from 'node:http'; - import type { BasicCrawlingContext, EnqueueLinksOptions, @@ -23,20 +21,19 @@ import { import type { Dictionary } from '@crawlee/types'; import { type CheerioRoot, type RobotsTxtFile, sleep } from '@crawlee/utils'; import * as cheerio from 'cheerio'; -// @ts-expect-error This throws a compilation error due to TypeScript not inferring the module has CJS versions too import { DOMParser } from 'linkedom/cached'; -import { concatStreamToBuffer } from '@apify/utilities'; - export type LinkeDOMErrorHandler< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler > = ErrorHandler>; export interface LinkeDOMCrawlerOptions< + ContextExtension = Dictionary, + ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends HttpCrawlerOptions> {} +> extends HttpCrawlerOptions, ContextExtension, ExtendedContext> {} export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit {} @@ -48,7 +45,7 @@ export type LinkeDOMHook< export interface LinkeDOMCrawlingContext< UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler -> extends InternalHttpCrawlingContext { +> extends InternalHttpCrawlingContext { window: Window; // Technically the document is not of type Document but of type either HTMLDocument or XMLDocument // from linkedom/types/{html/xml}/document, depending on the content type of the response @@ -121,11 +118,11 @@ export type LinkeDOMRequestHandler< * * The crawler finishes when there are no more {@apilink Request} objects to crawl. * - * We can use the `preNavigationHooks` to adjust `gotOptions`: + * We can use the `preNavigationHooks` to adjust the crawling context before the request is made: * * ``` * preNavigationHooks: [ - * (crawlingContext, gotOptions) => { + * (crawlingContext) => { * // ... * }, * ] @@ -163,17 +160,30 @@ export type LinkeDOMRequestHandler< * @category Crawlers */ -export class LinkeDOMCrawler extends HttpCrawler { +export class LinkeDOMCrawler< + ContextExtension = Dictionary, + ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, +> extends HttpCrawler { private static parser = new DOMParser(); - protected override async _parseHTML( - response: IncomingMessage, - isXml: boolean, - crawlingContext: LinkeDOMCrawlingContext, - ) { - const body = await concatStreamToBuffer(response); + constructor(options: LinkeDOMCrawlerOptions) { + super({ + ...options, + contextPipelineBuilder: () => + this.buildContextPipeline() + .compose({ + action: async (context) => this.parseContent(context), + }) + .compose({ action: async (context) => this.addHelpers(context) }), + }); + } - const document = LinkeDOMCrawler.parser.parseFromString(body.toString(), isXml ? 'text/xml' : 'text/html'); + private async parseContent(crawlingContext: InternalHttpCrawlingContext) { + const isXml = crawlingContext.contentType.type.includes('xml'); + const document = LinkeDOMCrawler.parser.parseFromString( + crawlingContext.body.toString(), + isXml ? 'text/xml' : 'text/html', + ); return { window: document.defaultView, @@ -184,6 +194,11 @@ export class LinkeDOMCrawler extends HttpCrawler { // See comment about typing in LinkeDOMCrawlingContext definition return document as unknown as Document; }, + }; + } + + private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { body: string }) { + return { enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => { return linkedomCrawlerEnqueueLinks({ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) }, @@ -195,34 +210,29 @@ export class LinkeDOMCrawler extends HttpCrawler { finalRequestUrl: crawlingContext.request.loadedUrl, }); }, - }; - } + async waitForSelector(selector: string, timeoutMs = 5_000) { + const $ = cheerio.load(crawlingContext.body); - override async _runRequestHandler(context: LinkeDOMCrawlingContext) { - context.waitForSelector = async (selector: string, timeoutMs = 5_000) => { - const $ = cheerio.load(context.body); + if ($(selector).get().length === 0) { + if (timeoutMs) { + await sleep(50); + await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0)); + return; + } - if ($(selector).get().length === 0) { - if (timeoutMs) { - await sleep(50); - await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0)); - return; + throw new Error(`Selector '${selector}' not found.`); } + }, + async parseWithCheerio(selector?: string, _timeoutMs = 5_000) { + const $ = cheerio.load(crawlingContext.body); - throw new Error(`Selector '${selector}' not found.`); - } - }; - context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => { - const $ = cheerio.load(context.body); - - if (selector && $(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } + if (selector && $(selector).get().length === 0) { + throw new Error(`Selector '${selector}' not found.`); + } - return $; + return $; + }, }; - - await super._runRequestHandler(context); } } diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json index aeaf4944aa1d..40b5ee42be21 100644 --- a/packages/memory-storage/package.json +++ b/packages/memory-storage/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/memory-storage", - "version": "3.15.3", + "version": "4.0.0", "description": "A simple in-memory storage implementation of the Apify API", "engines": { - "node": ">= 16" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -41,22 +35,22 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/log": "^2.4.0", - "@crawlee/types": "3.15.3", - "@sapphire/async-queue": "^1.5.0", - "@sapphire/shapeshift": "^3.0.0", - "content-type": "^1.0.4", - "fs-extra": "^11.0.0", + "@apify/log": "^2.5.18", + "@crawlee/types": "4.0.0", + "@sapphire/async-queue": "^1.5.5", + "@sapphire/shapeshift": "^4.0.0", + "content-type": "^1.0.5", + "fs-extra": "^11.3.0", "json5": "^2.2.3", - "mime-types": "^2.1.35", + "mime-types": "^3.0.1", "proper-lockfile": "^4.1.2", - "tslib": "^2.4.0" + "tslib": "^2.8.1" } } diff --git a/packages/memory-storage/src/background-handler/fs-utils.ts b/packages/memory-storage/src/background-handler/fs-utils.ts index 3e0bb1ba4d28..387784cecaa7 100644 --- a/packages/memory-storage/src/background-handler/fs-utils.ts +++ b/packages/memory-storage/src/background-handler/fs-utils.ts @@ -3,12 +3,12 @@ import { writeFile as writeFileP } from 'node:fs/promises'; import { resolve } from 'node:path'; import { setTimeout } from 'node:timers/promises'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; import { lock } from 'proper-lockfile'; import log from '@apify/log'; -import type { BackgroundHandlerReceivedMessage, BackgroundHandlerUpdateMetadataMessage } from '../utils'; +import type { BackgroundHandlerReceivedMessage, BackgroundHandlerUpdateMetadataMessage } from '../utils.js'; const backgroundHandlerLog = log.child({ prefix: 'MemoryStorageBackgroundHandler' }); diff --git a/packages/memory-storage/src/background-handler/index.ts b/packages/memory-storage/src/background-handler/index.ts index 4f2c1ee02726..5912cc7ad181 100644 --- a/packages/memory-storage/src/background-handler/index.ts +++ b/packages/memory-storage/src/background-handler/index.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; -import type { BackgroundHandlerReceivedMessage } from '../utils'; -import { handleMessage } from './fs-utils'; +import type { BackgroundHandlerReceivedMessage } from '../utils.js'; +import { handleMessage } from './fs-utils.js'; /** * A map of promises that are created when a background task is scheduled. diff --git a/packages/memory-storage/src/cache-helpers.ts b/packages/memory-storage/src/cache-helpers.ts index 157e69847c6c..6a1bc4bb8ce6 100644 --- a/packages/memory-storage/src/cache-helpers.ts +++ b/packages/memory-storage/src/cache-helpers.ts @@ -5,10 +5,10 @@ import type * as storage from '@crawlee/types'; import json5 from 'json5'; import mimeTypes from 'mime-types'; -import { DatasetFileSystemEntry } from './fs/dataset/fs'; -import { KeyValueFileSystemEntry } from './fs/key-value-store/fs'; -import { RequestQueueFileSystemEntry } from './fs/request-queue/fs'; -import { type MemoryStorage } from './memory-storage'; +import { DatasetFileSystemEntry } from './fs/dataset/fs.js'; +import { KeyValueFileSystemEntry } from './fs/key-value-store/fs.js'; +import { RequestQueueFileSystemEntry } from './fs/request-queue/fs.js'; +import { type MemoryStorage } from './memory-storage.js'; const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; @@ -388,8 +388,8 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryN } /* eslint-disable import/first -- Fixing circulars */ -import { DatasetClient } from './resource-clients/dataset'; -import type { InternalKeyRecord } from './resource-clients/key-value-store'; -import { KeyValueStoreClient } from './resource-clients/key-value-store'; -import { RequestQueueClient } from './resource-clients/request-queue'; -import { memoryStorageLog } from './utils'; +import { DatasetClient } from './resource-clients/dataset.js'; +import type { InternalKeyRecord } from './resource-clients/key-value-store.js'; +import { KeyValueStoreClient } from './resource-clients/key-value-store.js'; +import { RequestQueueClient } from './resource-clients/request-queue.js'; +import { memoryStorageLog } from './utils.js'; diff --git a/packages/memory-storage/src/fs/dataset/fs.ts b/packages/memory-storage/src/fs/dataset/fs.ts index 06bf21d5d40a..bf66d2d5ed84 100644 --- a/packages/memory-storage/src/fs/dataset/fs.ts +++ b/packages/memory-storage/src/fs/dataset/fs.ts @@ -2,11 +2,11 @@ import { readFile, rm } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { AsyncQueue } from '@sapphire/async-queue'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; -import { lockAndWrite } from '../../background-handler/fs-utils'; -import type { StorageImplementation } from '../common'; -import type { CreateStorageImplementationOptions } from './index'; +import { lockAndWrite } from '../../background-handler/fs-utils.js'; +import type { StorageImplementation } from '../common.js'; +import type { CreateStorageImplementationOptions } from './index.js'; export class DatasetFileSystemEntry implements StorageImplementation { private filePath: string; diff --git a/packages/memory-storage/src/fs/dataset/index.ts b/packages/memory-storage/src/fs/dataset/index.ts index 3fc24562fa4c..2d61c910dcdd 100644 --- a/packages/memory-storage/src/fs/dataset/index.ts +++ b/packages/memory-storage/src/fs/dataset/index.ts @@ -1,8 +1,8 @@ import type { Dictionary } from '@crawlee/types'; -import type { StorageImplementation } from '../common'; -import { DatasetFileSystemEntry } from './fs'; -import { DatasetMemoryEntry } from './memory'; +import type { StorageImplementation } from '../common.js'; +import { DatasetFileSystemEntry } from './fs.js'; +import { DatasetMemoryEntry } from './memory.js'; export function createDatasetStorageImplementation( options: CreateStorageImplementationOptions, diff --git a/packages/memory-storage/src/fs/dataset/memory.ts b/packages/memory-storage/src/fs/dataset/memory.ts index 569b77beb337..c0a7e6f475e1 100644 --- a/packages/memory-storage/src/fs/dataset/memory.ts +++ b/packages/memory-storage/src/fs/dataset/memory.ts @@ -1,4 +1,4 @@ -import type { StorageImplementation } from '../common'; +import type { StorageImplementation } from '../common.js'; export class DatasetMemoryEntry implements StorageImplementation { private data!: Data; diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.ts index 48b727d639ca..880c70974335 100644 --- a/packages/memory-storage/src/fs/key-value-store/fs.ts +++ b/packages/memory-storage/src/fs/key-value-store/fs.ts @@ -3,14 +3,14 @@ import { dirname, resolve } from 'node:path'; import { basename } from 'node:path/win32'; import { AsyncQueue } from '@sapphire/async-queue'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; import mime from 'mime-types'; -import { lockAndWrite } from '../../background-handler/fs-utils'; -import type { InternalKeyRecord } from '../../resource-clients/key-value-store'; -import { memoryStorageLog } from '../../utils'; -import type { StorageImplementation } from '../common'; -import type { CreateStorageImplementationOptions } from '.'; +import { lockAndWrite } from '../../background-handler/fs-utils.js'; +import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js'; +import { memoryStorageLog } from '../../utils.js'; +import type { StorageImplementation } from '../common.js'; +import type { CreateStorageImplementationOptions } from './index.js'; export class KeyValueFileSystemEntry implements StorageImplementation { private storeDirectory: string; @@ -34,8 +34,9 @@ export class KeyValueFileSystemEntry implements StorageImplementation { private data!: InternalKeyRecord; diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.ts index 23a4d8ff2ee8..87dde6a2163b 100644 --- a/packages/memory-storage/src/fs/request-queue/fs.ts +++ b/packages/memory-storage/src/fs/request-queue/fs.ts @@ -2,12 +2,12 @@ import { readFile, rm } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { AsyncQueue } from '@sapphire/async-queue'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; -import { lockAndCallback, lockAndWrite } from '../../background-handler/fs-utils'; -import type { InternalRequest } from '../../resource-clients/request-queue'; -import type { StorageImplementation } from '../common'; -import type { CreateStorageImplementationOptions } from '.'; +import { lockAndCallback, lockAndWrite } from '../../background-handler/fs-utils.js'; +import type { InternalRequest } from '../../resource-clients/request-queue.js'; +import type { StorageImplementation } from '../common.js'; +import type { CreateStorageImplementationOptions } from './index.js'; export class RequestQueueFileSystemEntry implements StorageImplementation { private filePath: string; diff --git a/packages/memory-storage/src/fs/request-queue/index.ts b/packages/memory-storage/src/fs/request-queue/index.ts index 25662a4fb921..cb903bb6e8c3 100644 --- a/packages/memory-storage/src/fs/request-queue/index.ts +++ b/packages/memory-storage/src/fs/request-queue/index.ts @@ -1,5 +1,5 @@ -import { RequestQueueFileSystemEntry } from './fs'; -import { RequestQueueMemoryEntry } from './memory'; +import { RequestQueueFileSystemEntry } from './fs.js'; +import { RequestQueueMemoryEntry } from './memory.js'; export function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions) { if (options.persistStorage) { diff --git a/packages/memory-storage/src/fs/request-queue/memory.ts b/packages/memory-storage/src/fs/request-queue/memory.ts index 79811781b30f..9399c2ed4f0b 100644 --- a/packages/memory-storage/src/fs/request-queue/memory.ts +++ b/packages/memory-storage/src/fs/request-queue/memory.ts @@ -1,5 +1,5 @@ -import type { InternalRequest } from '../../resource-clients/request-queue'; -import type { StorageImplementation } from '../common'; +import type { InternalRequest } from '../../resource-clients/request-queue.js'; +import type { StorageImplementation } from '../common.js'; export class RequestQueueMemoryEntry implements StorageImplementation { private data!: InternalRequest; diff --git a/packages/memory-storage/src/index.ts b/packages/memory-storage/src/index.ts index 6231f1fc1789..63137d81493b 100644 --- a/packages/memory-storage/src/index.ts +++ b/packages/memory-storage/src/index.ts @@ -1 +1 @@ -export * from './memory-storage'; +export * from './memory-storage.js'; diff --git a/packages/memory-storage/src/memory-storage.ts b/packages/memory-storage/src/memory-storage.ts index c19931dbd07f..5ba10b597add 100644 --- a/packages/memory-storage/src/memory-storage.ts +++ b/packages/memory-storage/src/memory-storage.ts @@ -5,15 +5,15 @@ import { resolve } from 'node:path'; import type * as storage from '@crawlee/types'; import type { Dictionary } from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { ensureDirSync, move, moveSync, pathExistsSync } from 'fs-extra'; +import { ensureDirSync, move, moveSync, pathExistsSync } from 'fs-extra/esm'; -import { promiseMap } from './background-handler/index'; -import { DatasetClient } from './resource-clients/dataset'; -import { DatasetCollectionClient } from './resource-clients/dataset-collection'; -import { KeyValueStoreClient } from './resource-clients/key-value-store'; -import { KeyValueStoreCollectionClient } from './resource-clients/key-value-store-collection'; -import { RequestQueueClient } from './resource-clients/request-queue'; -import { RequestQueueCollectionClient } from './resource-clients/request-queue-collection'; +import { promiseMap } from './background-handler/index.js'; +import { DatasetClient } from './resource-clients/dataset.js'; +import { DatasetCollectionClient } from './resource-clients/dataset-collection.js'; +import { KeyValueStoreClient } from './resource-clients/key-value-store.js'; +import { KeyValueStoreCollectionClient } from './resource-clients/key-value-store-collection.js'; +import { RequestQueueClient } from './resource-clients/request-queue.js'; +import { RequestQueueCollectionClient } from './resource-clients/request-queue-collection.js'; export interface MemoryStorageOptions { /** @@ -51,9 +51,9 @@ export class MemoryStorage implements storage.StorageClient { constructor(options: MemoryStorageOptions = {}) { s.object({ - localDataDirectory: s.string.optional, - writeMetadata: s.boolean.optional, - persistStorage: s.boolean.optional, + localDataDirectory: s.string().optional(), + writeMetadata: s.boolean().optional(), + persistStorage: s.boolean().optional(), }).parse(options); // v3.0.0 used `crawlee_storage` as the default, we changed this in v3.0.1 to just `storage`, @@ -91,7 +91,7 @@ export class MemoryStorage implements storage.StorageClient { } dataset(id: string): storage.DatasetClient { - s.string.parse(id); + s.string().parse(id); return new DatasetClient({ id, baseStorageDirectory: this.datasetsDirectory, client: this }); } @@ -104,7 +104,7 @@ export class MemoryStorage implements storage.StorageClient { } keyValueStore(id: string): storage.KeyValueStoreClient { - s.string.parse(id); + s.string().parse(id); return new KeyValueStoreClient({ id, baseStorageDirectory: this.keyValueStoresDirectory, client: this }); } @@ -117,10 +117,10 @@ export class MemoryStorage implements storage.StorageClient { } requestQueue(id: string, options: storage.RequestQueueOptions = {}): storage.RequestQueueClient { - s.string.parse(id); + s.string().parse(id); s.object({ - clientKey: s.string.optional, - timeoutSecs: s.number.optional, + clientKey: s.string().optional(), + timeoutSecs: s.number().optional(), }).parse(options); return new RequestQueueClient({ @@ -132,9 +132,9 @@ export class MemoryStorage implements storage.StorageClient { } async setStatusMessage(message: string, options: storage.SetStatusMessageOptions = {}): Promise { - s.string.parse(message); + s.string().parse(message); s.object({ - isStatusMessageTerminal: s.boolean.optional, + isStatusMessageTerminal: s.boolean().optional(), }).parse(options); return Promise.resolve(); diff --git a/packages/memory-storage/src/resource-clients/common/base-client.ts b/packages/memory-storage/src/resource-clients/common/base-client.ts index 2ac882552f48..3385ae3ba3e9 100644 --- a/packages/memory-storage/src/resource-clients/common/base-client.ts +++ b/packages/memory-storage/src/resource-clients/common/base-client.ts @@ -1,4 +1,4 @@ -import type { StorageTypes } from '../../consts'; +import type { StorageTypes } from '../../consts.js'; export class BaseClient { id: string; diff --git a/packages/memory-storage/src/resource-clients/dataset-collection.ts b/packages/memory-storage/src/resource-clients/dataset-collection.ts index b82c8a262ce6..4aa2462aa9f7 100644 --- a/packages/memory-storage/src/resource-clients/dataset-collection.ts +++ b/packages/memory-storage/src/resource-clients/dataset-collection.ts @@ -3,10 +3,10 @@ import { resolve } from 'node:path'; import type * as storage from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { scheduleBackgroundTask } from '../background-handler'; -import { findOrCacheDatasetByPossibleId } from '../cache-helpers'; -import type { MemoryStorage } from '../index'; -import { DatasetClient } from './dataset'; +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { findOrCacheDatasetByPossibleId } from '../cache-helpers.js'; +import type { MemoryStorage } from '../index.js'; +import { DatasetClient } from './dataset.js'; export interface DatasetCollectionClientOptions { baseStorageDirectory: string; @@ -36,7 +36,7 @@ export class DatasetCollectionClient implements storage.DatasetCollectionClient } async getOrCreate(name?: string): Promise { - s.string.optional.parse(name); + s.string().optional().parse(name); if (name) { const found = await findOrCacheDatasetByPossibleId(this.client, name); diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.ts index a90df844a125..374aca59cdb6 100644 --- a/packages/memory-storage/src/resource-clients/dataset.ts +++ b/packages/memory-storage/src/resource-clients/dataset.ts @@ -6,15 +6,15 @@ import { resolve } from 'node:path'; import type { Dictionary } from '@crawlee/types'; import type * as storage from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { move } from 'fs-extra'; +import { move } from 'fs-extra/esm'; -import { scheduleBackgroundTask } from '../background-handler'; -import { findOrCacheDatasetByPossibleId } from '../cache-helpers'; -import { StorageTypes } from '../consts'; -import type { StorageImplementation } from '../fs/common'; -import { createDatasetStorageImplementation } from '../fs/dataset'; -import type { MemoryStorage } from '../index'; -import { BaseClient } from './common/base-client'; +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { findOrCacheDatasetByPossibleId } from '../cache-helpers.js'; +import { StorageTypes } from '../consts.js'; +import type { StorageImplementation } from '../fs/common.js'; +import { createDatasetStorageImplementation } from '../fs/dataset/index.js'; +import type { MemoryStorage } from '../index.js'; +import { BaseClient } from './common/base-client.js'; /** * This is what API returns in the x-apify-pagination-limit @@ -70,7 +70,7 @@ export class DatasetClient async update(newFields: storage.DatasetClientUpdateOptions = {}): Promise { const parsed = s .object({ - name: s.string.lengthGreaterThan(0).optional, + name: s.string().lengthGreaterThan(0).optional(), }) .parse(newFields); @@ -135,9 +135,9 @@ export class DatasetClient desc, } = s .object({ - desc: s.boolean.optional, - limit: s.number.int.optional, - offset: s.number.int.optional, + desc: s.boolean().optional(), + limit: s.number().int().optional(), + offset: s.number().int().optional(), }) .parse(options); @@ -174,11 +174,11 @@ export class DatasetClient async pushItems(items: string | Data | string[] | Data[]): Promise { const rawItems = s - .union( - s.string, - s.object({} as Data).passthrough, - s.array(s.union(s.string, s.object({} as Data).passthrough)), - ) + .union([ + s.string(), + s.object({} as Data).passthrough(), + s.array(s.union([s.string(), s.object({} as Data).passthrough()])), + ]) .parse(items) as Data[]; // Check by id diff --git a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts b/packages/memory-storage/src/resource-clients/key-value-store-collection.ts index d552374beb3a..f656b9019924 100644 --- a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts +++ b/packages/memory-storage/src/resource-clients/key-value-store-collection.ts @@ -3,10 +3,10 @@ import { resolve } from 'node:path'; import type * as storage from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { scheduleBackgroundTask } from '../background-handler'; -import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers'; -import type { MemoryStorage } from '../index'; -import { KeyValueStoreClient } from './key-value-store'; +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers.js'; +import type { MemoryStorage } from '../index.js'; +import { KeyValueStoreClient } from './key-value-store.js'; export interface KeyValueStoreCollectionClientOptions { baseStorageDirectory: string; @@ -36,7 +36,7 @@ export class KeyValueStoreCollectionClient implements storage.KeyValueStoreColle } async getOrCreate(name?: string): Promise { - s.string.optional.parse(name); + s.string().optional().parse(name); if (name) { const found = await findOrCacheKeyValueStoreByPossibleId(this.client, name); diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts index 1a3af0432b4f..d14f7db75852 100644 --- a/packages/memory-storage/src/resource-clients/key-value-store.ts +++ b/packages/memory-storage/src/resource-clients/key-value-store.ts @@ -5,18 +5,18 @@ import { Readable } from 'node:stream'; import type * as storage from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { move } from 'fs-extra'; +import { move } from 'fs-extra/esm'; import mime from 'mime-types'; -import { scheduleBackgroundTask } from '../background-handler'; -import { maybeParseBody } from '../body-parser'; -import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers'; -import { DEFAULT_API_PARAM_LIMIT, StorageTypes } from '../consts'; -import type { StorageImplementation } from '../fs/common'; -import { createKeyValueStorageImplementation } from '../fs/key-value-store'; -import type { MemoryStorage } from '../index'; -import { isBuffer, isStream } from '../utils'; -import { BaseClient } from './common/base-client'; +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { maybeParseBody } from '../body-parser.js'; +import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers.js'; +import { DEFAULT_API_PARAM_LIMIT, StorageTypes } from '../consts.js'; +import type { StorageImplementation } from '../fs/common.js'; +import { createKeyValueStorageImplementation } from '../fs/key-value-store/index.js'; +import type { MemoryStorage } from '../index.js'; +import { isBuffer, isStream } from '../utils.js'; +import { BaseClient } from './common/base-client.js'; const DEFAULT_LOCAL_FILE_EXTENSION = 'bin'; @@ -32,6 +32,7 @@ export interface InternalKeyRecord { value: Buffer | string; contentType?: string; extension: string; + filePath?: string; } export class KeyValueStoreClient extends BaseClient { @@ -65,7 +66,7 @@ export class KeyValueStoreClient extends BaseClient { async update(newFields: storage.KeyValueStoreClientUpdateOptions = {}): Promise { const parsed = s .object({ - name: s.string.lengthGreaterThan(0).optional, + name: s.string().lengthGreaterThan(0).optional(), }) .parse(newFields); @@ -125,10 +126,10 @@ export class KeyValueStoreClient extends BaseClient { prefix, } = s .object({ - limit: s.number.greaterThan(0).optional, - exclusiveStartKey: s.string.optional, - collection: s.string.optional, // This is ignored, but kept for validation consistency with API client. - prefix: s.string.optional, + limit: s.number().greaterThan(0).optional(), + exclusiveStartKey: s.string().optional(), + collection: s.string().optional(), // This is ignored, but kept for validation consistency with API client. + prefix: s.string().optional(), }) .parse(options); @@ -184,6 +185,27 @@ export class KeyValueStoreClient extends BaseClient { }; } + /** + * Generates a public file:// URL for accessing a specific record in the key-value store. + * + * Returns `undefined` if the record does not exist or has no associated file path (i.e., it is not stored as a file). + * @param key The key of the record to generate the public URL for. + */ + async getRecordPublicUrl(key: string): Promise { + s.string().parse(key); + + // Check by id + const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id); + + if (!existingStoreById) { + this.throwOnNonExisting(StorageTypes.KeyValueStore); + } + + const storageEntry = await existingStoreById.keyValueEntries.get(key)?.get(); + + return storageEntry?.filePath; + } + /** * Tests whether a record with the given key exists in the key-value store without retrieving its value. * @@ -191,7 +213,7 @@ export class KeyValueStoreClient extends BaseClient { * @returns `true` if the record exists, `false` if it does not. */ async recordExists(key: string): Promise { - s.string.parse(key); + s.string().parse(key); // Check by id const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id); @@ -207,13 +229,13 @@ export class KeyValueStoreClient extends BaseClient { key: string, options: storage.KeyValueStoreClientGetRecordOptions = {}, ): Promise { - s.string.parse(key); + s.string().parse(key); s.object({ - buffer: s.boolean.optional, + buffer: s.boolean().optional(), // These options are ignored, but kept here // for validation consistency with API client. - stream: s.boolean.optional, - disableRedirect: s.boolean.optional, + stream: s.boolean().optional(), + disableRedirect: s.boolean().optional(), }).parse(options); // Check by id @@ -252,11 +274,11 @@ export class KeyValueStoreClient extends BaseClient { async setRecord(record: storage.KeyValueStoreRecord): Promise { s.object({ - key: s.string.lengthGreaterThan(0), - value: s.union( - s.null, - s.string, - s.number, + key: s.string().lengthGreaterThan(0), + value: s.union([ + s.null(), + s.string(), + s.number(), s.instance(Buffer), s.instance(ArrayBuffer), s.typedArray(), @@ -264,8 +286,8 @@ export class KeyValueStoreClient extends BaseClient { s .object({}) .setValidationEnabled(false), - ), - contentType: s.string.lengthGreaterThan(0).optional, + ]), + contentType: s.string().lengthGreaterThan(0).optional(), }).parse(record); // Check by id @@ -330,7 +352,7 @@ export class KeyValueStoreClient extends BaseClient { } async deleteRecord(key: string): Promise { - s.string.parse(key); + s.string().parse(key); // Check by id const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id); diff --git a/packages/memory-storage/src/resource-clients/request-queue-collection.ts b/packages/memory-storage/src/resource-clients/request-queue-collection.ts index 004fd3aacbfa..81c4f634da97 100644 --- a/packages/memory-storage/src/resource-clients/request-queue-collection.ts +++ b/packages/memory-storage/src/resource-clients/request-queue-collection.ts @@ -3,10 +3,10 @@ import { resolve } from 'node:path'; import type * as storage from '@crawlee/types'; import { s } from '@sapphire/shapeshift'; -import { scheduleBackgroundTask } from '../background-handler'; -import { findRequestQueueByPossibleId } from '../cache-helpers'; -import type { MemoryStorage } from '../index'; -import { RequestQueueClient } from './request-queue'; +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { findRequestQueueByPossibleId } from '../cache-helpers.js'; +import type { MemoryStorage } from '../index.js'; +import { RequestQueueClient } from './request-queue.js'; export interface RequestQueueCollectionClientOptions { baseStorageDirectory: string; @@ -36,7 +36,7 @@ export class RequestQueueCollectionClient implements storage.RequestQueueCollect } async getOrCreate(name?: string): Promise { - s.string.optional.parse(name); + s.string().optional().parse(name); if (name) { const found = await findRequestQueueByPossibleId(this.client, name); diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts index a5cba68f5356..a5a1e388a7c4 100644 --- a/packages/memory-storage/src/resource-clients/request-queue.ts +++ b/packages/memory-storage/src/resource-clients/request-queue.ts @@ -5,33 +5,35 @@ import { resolve } from 'node:path'; import type * as storage from '@crawlee/types'; import { AsyncQueue } from '@sapphire/async-queue'; import { s } from '@sapphire/shapeshift'; -import { move } from 'fs-extra'; -import type { RequestQueueFileSystemEntry } from 'packages/memory-storage/src/fs/request-queue/fs'; -import type { RequestQueueMemoryEntry } from 'packages/memory-storage/src/fs/request-queue/memory'; - -import { scheduleBackgroundTask } from '../background-handler'; -import { findRequestQueueByPossibleId } from '../cache-helpers'; -import { StorageTypes } from '../consts'; -import { createRequestQueueStorageImplementation } from '../fs/request-queue'; -import type { MemoryStorage } from '../index'; -import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils'; -import { BaseClient } from './common/base-client'; - -const requestShape = s.object({ - id: s.string, - url: s.string.url({ allowedProtocols: ['http:', 'https:'] }), - uniqueKey: s.string, - method: s.string.optional, - retryCount: s.number.int.optional, - handledAt: s.union(s.string, s.date.valid).optional, -}).passthrough; +import { move } from 'fs-extra/esm'; +import type { RequestQueueFileSystemEntry } from 'packages/memory-storage/src/fs/request-queue/fs.js'; +import type { RequestQueueMemoryEntry } from 'packages/memory-storage/src/fs/request-queue/memory.js'; + +import { scheduleBackgroundTask } from '../background-handler/index.js'; +import { findRequestQueueByPossibleId } from '../cache-helpers.js'; +import { StorageTypes } from '../consts.js'; +import { createRequestQueueStorageImplementation } from '../fs/request-queue/index.js'; +import type { MemoryStorage } from '../index.js'; +import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js'; +import { BaseClient } from './common/base-client.js'; + +const requestShape = s + .object({ + id: s.string(), + url: s.string().url({ allowedProtocols: ['http:', 'https:'] }), + uniqueKey: s.string(), + method: s.string().optional(), + retryCount: s.number().int().optional(), + handledAt: s.union([s.string(), s.date().valid()]).optional(), + }) + .passthrough(); const requestShapeWithoutId = requestShape.omit(['id']); -const batchRequestShapeWithoutId = requestShapeWithoutId.array; +const batchRequestShapeWithoutId = requestShapeWithoutId.array(); const requestOptionsShape = s.object({ - forefront: s.boolean.optional, + forefront: s.boolean().optional(), }); export interface RequestQueueClientOptions { @@ -100,9 +102,10 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue // when swapping to a remote queue in production. const parsed = s .object({ - name: s.string.lengthGreaterThan(0).optional, + name: s.string().lengthGreaterThan(0).optional(), }) - .passthrough.parse(newFields); + .passthrough() + .parse(newFields); const existingQueueById = await findRequestQueueByPossibleId(this.client, this.name ?? this.id); @@ -166,7 +169,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue async listHead(options: storage.ListOptions = {}): Promise { const { limit } = s .object({ - limit: s.number.optional.default(100), + limit: s.number().optional().default(100), }) .parse(options); @@ -229,8 +232,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue async listAndLockHead(options: storage.ListAndLockOptions): Promise { const { limit, lockSecs } = s .object({ - limit: s.number.lessThanOrEqual(25).optional.default(25), - lockSecs: s.number, + limit: s.number().lessThanOrEqual(25).optional().default(25), + lockSecs: s.number(), }) .parse(options); @@ -302,11 +305,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue id: string, options: storage.ProlongRequestLockOptions, ): Promise { - s.string.parse(id); + s.string().parse(id); const { lockSecs, forefront } = s .object({ - lockSecs: s.number, - forefront: s.boolean.optional.default(false), + lockSecs: s.number(), + forefront: s.boolean().optional().default(false), }) .parse(options); @@ -337,10 +340,10 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue } async deleteRequestLock(id: string, options: storage.DeleteRequestLockOptions = {}): Promise { - s.string.parse(id); + s.string().parse(id); const { forefront } = s .object({ - forefront: s.boolean.optional.default(false), + forefront: s.boolean().optional().default(false), }) .parse(options); @@ -498,7 +501,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue } async getRequest(id: string): Promise { - s.string.parse(id); + s.string().parse(id); const queue = await this.getQueue(); const json = (await queue.requests.get(id)?.get())?.json; return this._jsonToRequest(json); diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.ts index b423bfb48aa7..04683d0dd088 100644 --- a/packages/memory-storage/src/utils.ts +++ b/packages/memory-storage/src/utils.ts @@ -5,7 +5,7 @@ import { s } from '@sapphire/shapeshift'; import defaultLog from '@apify/log'; -import { REQUEST_ID_LENGTH } from './consts'; +import { REQUEST_ID_LENGTH } from './consts.js'; /** * Removes all properties with a null value @@ -35,7 +35,7 @@ export function uniqueKeyToRequestId(uniqueKey: string): string { export function isBuffer(value: unknown): boolean { try { - s.union(s.instance(Buffer), s.instance(ArrayBuffer), s.typedArray()).parse(value); + s.union([s.instance(Buffer), s.instance(ArrayBuffer), s.typedArray()]).parse(value); return true; } catch { diff --git a/packages/memory-storage/test/fs-fallback.test.ts b/packages/memory-storage/test/fs-fallback.test.ts index 1f014d936bc1..50edb321edeb 100644 --- a/packages/memory-storage/test/fs-fallback.test.ts +++ b/packages/memory-storage/test/fs-fallback.test.ts @@ -4,10 +4,10 @@ import { resolve } from 'node:path'; import { MemoryStorage } from '@crawlee/memory-storage'; import type { KeyValueStoreRecord } from '@crawlee/types'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; describe('fallback to fs for reading', () => { - const tmpLocation = resolve(__dirname, './tmp/fs-fallback'); + const tmpLocation = resolve(import.meta.dirname, './tmp/fs-fallback'); const storage = new MemoryStorage({ localDataDirectory: tmpLocation, }); diff --git a/packages/memory-storage/test/key-value-store/with-extension.test.ts b/packages/memory-storage/test/key-value-store/with-extension.test.ts index 95595f79b410..ed29d2ce946e 100644 --- a/packages/memory-storage/test/key-value-store/with-extension.test.ts +++ b/packages/memory-storage/test/key-value-store/with-extension.test.ts @@ -1,8 +1,9 @@ +import { existsSync } from 'node:fs'; import { resolve } from 'node:path'; -import { emptyDirSync, existsSync } from 'fs-extra'; +import { emptyDirSync } from 'fs-extra/esm'; -import { createKeyValueStorageImplementation } from '../../src/fs/key-value-store'; +import { createKeyValueStorageImplementation } from '../../src/fs/key-value-store/index.js'; describe('KeyValueStore should append extension only when needed', () => { const mockImageBuffer = Buffer.from('This is a test image', 'utf8'); diff --git a/packages/memory-storage/test/no-crash-on-big-buffers.test.ts b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts index 67ae80fc2676..5677d20ac0f2 100644 --- a/packages/memory-storage/test/no-crash-on-big-buffers.test.ts +++ b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts @@ -8,7 +8,7 @@ import { MemoryStorage } from '@crawlee/memory-storage'; import type { KeyValueStoreClient, KeyValueStoreInfo } from '@crawlee/types'; describe('MemoryStorage should not crash when saving a big buffer', () => { - const tmpLocation = resolve(__dirname, './tmp/no-buffer-crash'); + const tmpLocation = resolve(import.meta.dirname, './tmp/no-buffer-crash'); const storage = new MemoryStorage({ localDataDirectory: tmpLocation, persistStorage: false, diff --git a/packages/memory-storage/test/no-writing-to-disk.test.ts b/packages/memory-storage/test/no-writing-to-disk.test.ts index e39fb7c9c46a..871bd689dc44 100644 --- a/packages/memory-storage/test/no-writing-to-disk.test.ts +++ b/packages/memory-storage/test/no-writing-to-disk.test.ts @@ -3,10 +3,10 @@ import { resolve } from 'node:path'; import { MemoryStorage } from '@crawlee/memory-storage'; -import { waitTillWrittenToDisk } from './__shared__'; +import { waitTillWrittenToDisk } from './__shared__.js'; describe('persistStorage option', () => { - const tmpLocation = resolve(__dirname, './tmp/no-writing-to-disk'); + const tmpLocation = resolve(import.meta.dirname, './tmp/no-writing-to-disk'); afterAll(async () => { await rm(tmpLocation, { force: true, recursive: true }); diff --git a/packages/memory-storage/test/request-queue/ignore-non-json-files.test.ts b/packages/memory-storage/test/request-queue/ignore-non-json-files.test.ts index a6ed41736da2..ee8543ee47c3 100644 --- a/packages/memory-storage/test/request-queue/ignore-non-json-files.test.ts +++ b/packages/memory-storage/test/request-queue/ignore-non-json-files.test.ts @@ -5,10 +5,10 @@ import { resolve } from 'node:path'; import { MemoryStorage } from '@crawlee/memory-storage'; import type { InternalRequest } from '@crawlee/memory-storage/src/resource-clients/request-queue'; import type { RequestSchema } from '@crawlee/types'; -import { ensureDir } from 'fs-extra'; +import { ensureDir } from 'fs-extra/esm'; describe('when falling back to fs, Request queue should ignore non-JSON files', () => { - const tmpLocation = resolve(__dirname, './tmp/req-queue-ignore-non-json'); + const tmpLocation = resolve(import.meta.dirname, './tmp/req-queue-ignore-non-json'); const storage = new MemoryStorage({ localDataDirectory: tmpLocation, }); diff --git a/packages/memory-storage/test/reverse-datataset-list.test.ts b/packages/memory-storage/test/reverse-datataset-list.test.ts index 4dee00ce8f8e..07cd85e9c011 100644 --- a/packages/memory-storage/test/reverse-datataset-list.test.ts +++ b/packages/memory-storage/test/reverse-datataset-list.test.ts @@ -7,7 +7,7 @@ import type { DatasetClient } from '@crawlee/types'; const elements = Array.from({ length: 10 }, (_, i) => ({ number: i })); describe('Dataset#listItems respects the desc option', () => { - const localDataDirectory = resolve(__dirname, './tmp/desc'); + const localDataDirectory = resolve(import.meta.dirname, './tmp/desc'); const storage = new MemoryStorage({ localDataDirectory, persistStorage: false, diff --git a/packages/memory-storage/test/write-metadata.test.ts b/packages/memory-storage/test/write-metadata.test.ts index eb36325950e9..f74203dc2346 100644 --- a/packages/memory-storage/test/write-metadata.test.ts +++ b/packages/memory-storage/test/write-metadata.test.ts @@ -3,10 +3,10 @@ import { resolve } from 'node:path'; import { MemoryStorage } from '@crawlee/memory-storage'; -import { waitTillWrittenToDisk } from './__shared__'; +import { waitTillWrittenToDisk } from './__shared__.js'; describe('writeMetadata option', () => { - const tmpLocation = resolve(__dirname, './tmp/write-metadata-tests'); + const tmpLocation = resolve(import.meta.dirname, './tmp/write-metadata-tests'); afterAll(async () => { await rm(tmpLocation, { force: true, recursive: true }); diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json index 70287c15c619..c1df4a895661 100644 --- a/packages/playwright-crawler/package.json +++ b/packages/playwright-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/playwright", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,29 +40,30 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/datastructures": "^2.0.0", - "@apify/log": "^2.4.0", - "@apify/timeout": "^0.3.1", - "@crawlee/browser": "3.15.3", - "@crawlee/browser-pool": "3.15.3", - "@crawlee/core": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "cheerio": "1.0.0-rc.12", - "jquery": "^3.6.0", - "lodash.isequal": "^4.5.0", + "@apify/datastructures": "^2.0.3", + "@apify/log": "^2.5.18", + "@apify/timeout": "^0.3.2", + "@crawlee/browser": "4.0.0", + "@crawlee/browser-pool": "4.0.0", + "@crawlee/cheerio": "4.0.0", + "@crawlee/core": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "cheerio": "^1.0.0", + "idcac-playwright": "^0.1.3", + "jquery": "^3.7.1", "ml-logistic-regression": "^2.0.0", - "ml-matrix": "^6.11.0", - "ow": "^0.28.1", + "ml-matrix": "^6.12.1", + "ow": "^2.0.0", "string-comparison": "^1.3.0", - "tslib": "^2.4.0" + "tslib": "^2.8.1" }, "peerDependencies": { "idcac-playwright": "^0.1.2", diff --git a/packages/playwright-crawler/src/index.ts b/packages/playwright-crawler/src/index.ts index 06c0490346ad..e745ea76b185 100644 --- a/packages/playwright-crawler/src/index.ts +++ b/packages/playwright-crawler/src/index.ts @@ -1,10 +1,10 @@ export * from '@crawlee/browser'; -export * from './internals/playwright-crawler'; -export * from './internals/playwright-launcher'; -export * from './internals/adaptive-playwright-crawler'; -export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction'; +export * from './internals/playwright-crawler.js'; +export * from './internals/playwright-launcher.js'; +export * from './internals/adaptive-playwright-crawler.js'; +export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js'; -export * as playwrightUtils from './internals/utils/playwright-utils'; -export * as playwrightClickElements from './internals/enqueue-links/click-elements'; -export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils'; -export type { RenderingType } from './internals/utils/rendering-type-prediction'; +export * as playwrightUtils from './internals/utils/playwright-utils.js'; +export * as playwrightClickElements from './internals/enqueue-links/click-elements.js'; +export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils.js'; +export type { RenderingType } from './internals/utils/rendering-type-prediction.js'; diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts index 3658dcb94d91..7d848019fbe1 100644 --- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts @@ -1,7 +1,13 @@ -import type { BrowserHook, LoadedContext, LoadedRequest, Request, RouterHandler } from '@crawlee/browser'; +import { isDeepStrictEqual } from 'node:util'; + +import { BasicCrawler } from '@crawlee/basic'; +import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser'; import { extractUrlsFromPage } from '@crawlee/browser'; +import type { CheerioCrawlingContext } from '@crawlee/cheerio'; +import { CheerioCrawler } from '@crawlee/cheerio'; import type { - BaseHttpResponseData, + ContextPipeline, + CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RequestQueue, @@ -13,26 +19,26 @@ import type { } from '@crawlee/core'; import { Configuration, + RequestHandlerError, RequestHandlerResult, - RequestState, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core'; -import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types'; +import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types'; import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils'; -import { type Cheerio, type Element, load } from 'cheerio'; -import isEqual from 'lodash.isequal'; +import { type Cheerio } from 'cheerio'; +import type { AnyNode } from 'domhandler'; import type { Page } from 'playwright'; import type { SetRequired } from 'type-fest'; import type { Log } from '@apify/log'; import { addTimeoutToPromise } from '@apify/timeout'; -import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler'; -import { PlaywrightCrawler } from './playwright-crawler'; -import { type RenderingType, RenderingTypePredictor } from './utils/rendering-type-prediction'; +import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js'; +import { PlaywrightCrawler } from './playwright-crawler.js'; +import { type RenderingType, RenderingTypePredictor } from './utils/rendering-type-prediction.js'; type Result = | { result: TResult; ok: true; logs?: LogProxyCall[] } @@ -97,11 +103,12 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics { } export interface AdaptivePlaywrightCrawlerContext - extends RestrictedCrawlingContext { + extends CrawlingContext { + request: LoadedRequest>; /** * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation. */ - response: BaseHttpResponseData; + response: Response; /** * Playwright Page object. If accessed in HTTP-only rendering, this will throw an error and make the AdaptivePlaywrightCrawlerContext retry the request in a browser. @@ -112,7 +119,7 @@ export interface AdaptivePlaywrightCrawlerContext>; + querySelector(selector: string, timeoutMs?: number): Promise>; /** * Wait for an element matching the selector to appear. @@ -142,32 +149,25 @@ export interface AdaptivePlaywrightCrawlerContext; + + enqueueLinks(options?: EnqueueLinksOptions): Promise; } interface AdaptiveHook extends BrowserHook< - Pick & { page?: Page }, + Pick & { + page?: Page; + request: Request; + }, PlaywrightGotoOptions > {} -export interface AdaptivePlaywrightCrawlerOptions - extends Omit< - PlaywrightCrawlerOptions, - 'requestHandler' | 'handlePageFunction' | 'preNavigationHooks' | 'postNavigationHooks' +export interface AdaptivePlaywrightCrawlerOptions< + ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, +> extends Omit< + BasicCrawlerOptions, + 'preNavigationHooks' | 'postNavigationHooks' > { - /** - * Function that is called to process each request. - * - * The function receives the {@apilink AdaptivePlaywrightCrawlingContext} as an argument, and it must refrain from calling code with side effects, - * other than the methods of the crawling context. Any other side effects may be invoked repeatedly by the crawler, which can lead to inconsistent results. - * - * The function must return a promise, which is then awaited by the crawler. - * - * If the function throws an exception, the crawler will try to re-crawl the - * request later, up to `option.maxRequestRetries` times. - */ - requestHandler?: (crawlingContext: LoadedContext) => Awaitable; - /** * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies. * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling, @@ -180,7 +180,7 @@ export interface AdaptivePlaywrightCrawlerOptions * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling, * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser. */ - postNavigationHooks?: AdaptiveHook[]; + postNavigationHooks?: AdaptiveHook[]; // TODO should contain a LoadedRequest - reflect that /** * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests. @@ -263,24 +263,23 @@ type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args * * @experimental */ -export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { - private adaptiveRequestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] & {}; +export class AdaptivePlaywrightCrawler< + ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, +> extends BasicCrawler { private renderingTypePredictor: NonNullable; private resultChecker: NonNullable; private resultComparator: NonNullable; private preventDirectStorageAccess: boolean; + private staticContextPipeline: ContextPipeline; + private browserContextPipeline: ContextPipeline; + private individualRequestHandlerTimeoutMillis: number; declare readonly stats: AdaptivePlaywrightCrawlerStatistics; + private resultObjects = new WeakMap(); - /** - * Default {@apilink Router} instance that will be used if we don't specify any {@apilink AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}. - * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}. - */ - // @ts-ignore - override readonly router: RouterHandler = - Router.create(); + private teardownHooks: (() => Promise)[] = []; constructor( - options: AdaptivePlaywrightCrawlerOptions = {}, + options: AdaptivePlaywrightCrawlerOptions = {}, override readonly config = Configuration.getGlobalConfig(), ) { const { @@ -291,11 +290,33 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { resultComparator, statisticsOptions, preventDirectStorageAccess = true, + requestHandlerTimeoutSecs = 60, + errorHandler, + failedRequestHandler, + preNavigationHooks, + postNavigationHooks, + extendContext, + contextPipelineBuilder, ...rest } = options; - super(rest, config); - this.adaptiveRequestHandler = requestHandler ?? this.router; + super( + { + ...rest, + // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers + errorHandler, + failedRequestHandler, + // Same for request handler + requestHandler, + // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines + // (that would be a logical error in this class) + contextPipelineBuilder: () => + null as unknown as ContextPipeline, + }, + config, + ); + this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000; + this.renderingTypePredictor = renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }); this.resultChecker = resultChecker ?? (() => true); @@ -310,11 +331,80 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { resultA.datasetItems.length === resultB.datasetItems.length && resultA.datasetItems.every((itemA, i) => { const itemB = resultB.datasetItems[i]; - return isEqual(itemA, itemB); + return isDeepStrictEqual(itemA, itemB); }) ); }; } + const staticCrawler = new CheerioCrawler( + { + ...rest, + useSessionPool: false, + statisticsOptions: { + persistenceOptions: { enable: false }, + }, + preNavigationHooks: [ + async (context) => { + for (const hook of preNavigationHooks ?? []) { + await hook(context, undefined); + } + }, + ], + postNavigationHooks: [ + async (context) => { + for (const hook of postNavigationHooks ?? []) { + await hook(context, undefined); + } + }, + ], + }, + config, + ); + + const browserCrawler = new PlaywrightCrawler( + { + ...rest, + useSessionPool: false, + statisticsOptions: { + persistenceOptions: { enable: false }, + }, + preNavigationHooks: [ + async (context, gotoOptions) => { + for (const hook of preNavigationHooks ?? []) { + await hook(context, gotoOptions); + } + }, + ], + postNavigationHooks: [ + async (context, gotoOptions) => { + for (const hook of postNavigationHooks ?? []) { + await hook(context, gotoOptions); + } + }, + ], + }, + config, + ); + + this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler)); + + this.staticContextPipeline = staticCrawler.contextPipeline + .compose({ + action: this.adaptCheerioContext.bind(this), + }) + .compose({ + action: async (context) => + extendContext ? await extendContext(context) : (context as unknown as ExtendedContext), + }); + + this.browserContextPipeline = browserCrawler.contextPipeline + .compose({ + action: this.adaptPlaywrightContext.bind(this), + }) + .compose({ + action: async (context) => + extendContext ? await extendContext(context) : (context as unknown as ExtendedContext), + }); this.stats = new AdaptivePlaywrightCrawlerStatistics({ logMessage: `${this.log.getOptions().prefix} request statistics:`, @@ -330,7 +420,149 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { return await super._init(); } - protected override async _runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise { + private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) { + // Capture the original response to avoid infinite recursion when the getter is copied to the context + const result = this.resultObjects.get(cheerioContext); + if (result === undefined) { + throw new Error('Logical error - `this.resultObjects` does not contain the result object'); + } + + return { + get page(): Page { + throw new Error('Page object was used in HTTP-only request handler'); + }, + async querySelector(selector: string) { + return cheerioContext.$(selector); + }, + enqueueLinks: async (options: EnqueueLinksOptions = {}) => { + const urls = + options.urls ?? + extractUrlsFromCheerio( + cheerioContext.$, + options.selector, + options.baseUrl ?? cheerioContext.request.loadedUrl, + ); + return (await this.enqueueLinks( + { ...options, urls }, + cheerioContext.request, + result, + )) as unknown as void; + }, + response: cheerioContext.response, + }; + } + + private async adaptPlaywrightContext(playwrightContext: PlaywrightCrawlingContext) { + const originalResponse = playwrightContext.response; + + const result = this.resultObjects.get(playwrightContext); + if (result === undefined) { + throw new Error('Logical error - `this.resultObjects` does not contain the result object'); + } + + return { + response: new Response(Uint8Array.from(await originalResponse.body()), { + headers: originalResponse.headers(), + status: originalResponse.status(), + statusText: originalResponse.statusText(), + }), + async querySelector(selector: string, timeoutMs = 5000) { + const locator = playwrightContext.page.locator(selector).first(); + await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); + const $ = await playwrightContext.parseWithCheerio(); + + return $(selector) as Cheerio; + }, + enqueueLinks: async (options: EnqueueLinksOptions = {}, timeoutMs = 5000) => { + // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication + let urls: readonly string[]; + + if (options.urls === undefined) { + const selector = options.selector ?? 'a'; + const locator = playwrightContext.page.locator(selector).first(); + await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); + urls = + options.urls ?? + (await extractUrlsFromPage( + playwrightContext.page, + selector, + options.baseUrl ?? playwrightContext.request.loadedUrl, + )); + } else { + urls = options.urls; + } + + return (await this.enqueueLinks( + { ...options, urls }, + playwrightContext.request, + result, + )) as unknown as void; + }, + }; + } + + private async crawlOne( + renderingType: RenderingType, + context: CrawlingContext, + useStateFunction: (defaultValue?: Dictionary) => Promise, + ): Promise> { + const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY); + const logs: LogProxyCall[] = []; + + const deferredCleanup: (() => Promise)[] = []; + + const resultBoundContextHelpers = { + addRequests: result.addRequests, + pushData: result.pushData, + useState: this.allowStorageAccess(useStateFunction), + getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore), + enqueueLinks: async (options: SetRequired) => { + return await this.enqueueLinks(options, context.request, result); + }, + log: this.createLogProxy(context.log, logs), + registerDeferredCleanup: (cleanup: () => Promise) => deferredCleanup.push(cleanup), + }; + + const subCrawlerContext = { ...context, ...resultBoundContextHelpers }; + this.resultObjects.set(subCrawlerContext, result); + + try { + const callAdaptiveRequestHandler = async () => { + if (renderingType === 'static') { + await this.staticContextPipeline.call( + subCrawlerContext, + async (finalContext) => await this.requestHandler(finalContext), + ); + } else if (renderingType === 'clientOnly') { + await this.browserContextPipeline.call( + subCrawlerContext, + async (finalContext) => await this.requestHandler(finalContext), + ); + } + }; + + await addTimeoutToPromise( + async () => + withCheckedStorageAccess(() => { + if (this.preventDirectStorageAccess) { + throw new Error( + 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler', + ); + } + }, callAdaptiveRequestHandler), + this.individualRequestHandlerTimeoutMillis, + 'Request handler timed out', + ); + + return { result, ok: true, logs }; + } catch (error) { + return { error, ok: false, logs }; + } finally { + await Promise.all(deferredCleanup.map((cleanup) => cleanup())); + } + } + + protected override async runRequestHandler(crawlingContext: CrawlingContext): Promise { const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request); const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation; @@ -344,7 +576,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`); this.stats.trackHttpOnlyRequestHandlerRun(); - const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext); + const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState); if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) { crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`); @@ -352,9 +584,16 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { await this.commitResult(crawlingContext, plainHTTPRun.result); return; } + + // Execution will "fall through" and try running the request handler in a browser if (!plainHTTPRun.ok) { + const actualError = + plainHTTPRun.error instanceof RequestHandlerError + ? (plainHTTPRun.error.cause as Error) + : (plainHTTPRun.error as Error); + crawlingContext.log.exception( - plainHTTPRun.error as Error, + actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`, ); } else { @@ -372,7 +611,30 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { // a rendering type detection if necessary. Without this measure, the HTTP request handler would run // under different conditions, which could change its behavior. Changes done to the crawler state by // the HTTP request handler will not be committed to the actual storage. - const { result: browserRun, initialStateCopy } = await this.runRequestHandlerInBrowser(crawlingContext); + const stateTracker = { + stateCopy: null, + async getLiveState(defaultValue: Dictionary = {}) { + const state = await crawlingContext.useState(defaultValue); + + if (this.stateCopy === null) { + this.stateCopy = JSON.parse(JSON.stringify(state)); + } + + return state; + }, + async getStateCopy(defaultValue: Dictionary = {}) { + if (this.stateCopy === null) { + return defaultValue; + } + return this.stateCopy; + }, + }; + + const browserRun = await this.crawlOne( + 'clientOnly', + crawlingContext, + stateTracker.getLiveState.bind(stateTracker), + ); if (!browserRun.ok) { throw browserRun.error; @@ -382,7 +644,11 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { if (shouldDetectRenderingType) { crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`); - const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext, initialStateCopy); + const plainHTTPRun = await this.crawlOne( + 'static', + crawlingContext, + stateTracker.getStateCopy.bind(stateTracker), + ); const detectionResult: RenderingType | undefined = (() => { if (!plainHTTPRun.ok) { @@ -410,7 +676,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { } protected async commitResult( - crawlingContext: PlaywrightCrawlingContext, + crawlingContext: CrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult, ): Promise { await Promise.all([ @@ -437,234 +703,6 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { ); } - protected async runRequestHandlerInBrowser( - crawlingContext: PlaywrightCrawlingContext, - ): Promise<{ result: Result; initialStateCopy?: Record }> { - const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY); - let initialStateCopy: Record | undefined; - - try { - await super._runRequestHandler.call( - new Proxy(this, { - get: (target, propertyName, receiver) => { - if (propertyName === 'userProvidedRequestHandler') { - return async (playwrightContext: PlaywrightCrawlingContext) => - withCheckedStorageAccess( - () => { - if (this.preventDirectStorageAccess) { - throw new Error( - 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler', - ); - } - }, - () => - this.adaptiveRequestHandler({ - id: crawlingContext.id, - session: crawlingContext.session, - proxyInfo: crawlingContext.proxyInfo, - request: crawlingContext.request as LoadedRequest, - response: { - url: crawlingContext.response!.url(), - statusCode: crawlingContext.response!.status(), - headers: crawlingContext.response!.headers(), - trailers: {}, - complete: true, - redirectUrls: [], - }, - log: crawlingContext.log, - page: crawlingContext.page, - querySelector: async (selector, timeoutMs = 5_000) => { - const locator = playwrightContext.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); - const $ = await playwrightContext.parseWithCheerio(); - - return $(selector) as Cheerio; - }, - async waitForSelector(selector, timeoutMs = 5_000) { - const locator = playwrightContext.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); - }, - async parseWithCheerio( - selector?: string, - timeoutMs = 5_000, - ): Promise { - if (selector) { - const locator = playwrightContext.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); - } - - return playwrightContext.parseWithCheerio(); - }, - enqueueLinks: async (options = {}, timeoutMs = 5_000) => { - let urls; - - if (options.urls === undefined) { - const selector = options.selector ?? 'a'; - const locator = playwrightContext.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); - - urls = await extractUrlsFromPage( - playwrightContext.page, - selector, - options.baseUrl ?? - playwrightContext.request.loadedUrl ?? - playwrightContext.request.url, - ); - } else { - urls = options.urls; - } - - return await this.enqueueLinks( - { ...options, urls }, - crawlingContext.request, - result, - ); - }, - addRequests: result.addRequests, - pushData: result.pushData, - useState: this.allowStorageAccess(async (defaultValue) => { - const state = await result.useState(defaultValue); - if (initialStateCopy === undefined) { - initialStateCopy = JSON.parse(JSON.stringify(state)); - } - return state; - }), - getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore), - }), - ); - } - return Reflect.get(target, propertyName, receiver); - }, - }), - crawlingContext, - ); - return { result: { result, ok: true }, initialStateCopy }; - } catch (error) { - return { result: { error, ok: false }, initialStateCopy }; - } - } - - protected async runRequestHandlerWithPlainHTTP( - crawlingContext: PlaywrightCrawlingContext, - oldStateCopy?: Dictionary, - ): Promise> { - const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY); - const logs: LogProxyCall[] = []; - - const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler - - try { - await withCheckedStorageAccess( - () => { - if (this.preventDirectStorageAccess) { - throw new Error( - 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler', - ); - } - }, - async () => - addTimeoutToPromise( - async () => { - const hookContext: Parameters[0] = { - id: crawlingContext.id, - session: crawlingContext.session, - proxyInfo: crawlingContext.proxyInfo, - request: crawlingContext.request, - log: this.createLogProxy(crawlingContext.log, logs), - }; - - await this._executeHooks( - this.preNavigationHooks, - { - ...hookContext, - get page(): Page { - throw new Error('Page object was used in HTTP-only pre-navigation hook'); - }, - } as PlaywrightCrawlingContext, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context - pageGotoOptions, - ); - - const response = await crawlingContext.sendRequest({}); - - const loadedUrl = response.url; - crawlingContext.request.loadedUrl = loadedUrl; - - if (!this.requestMatchesEnqueueStrategy(crawlingContext.request)) { - const request = crawlingContext.request; - - this.log.debug( - // eslint-disable-next-line dot-notation - `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`, - ); - - request.noRetry = true; - request.state = RequestState.SKIPPED; - - await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); - - return; - } - - const $ = load(response.body); - - await this.adaptiveRequestHandler({ - ...hookContext, - request: crawlingContext.request as LoadedRequest, - response, - get page(): Page { - throw new Error('Page object was used in HTTP-only request handler'); - }, - async querySelector(selector, _timeoutMs?: number) { - return $(selector) as Cheerio; - }, - async waitForSelector(selector, _timeoutMs?: number) { - if ($(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } - }, - async parseWithCheerio(selector?: string, _timeoutMs?: number): Promise { - if (selector && $(selector).get().length === 0) { - throw new Error(`Selector '${selector}' not found.`); - } - - return $; - }, - enqueueLinks: async ( - options: Parameters[0] = {}, - ) => { - const urls = - options.urls ?? - extractUrlsFromCheerio($, options.selector, options.baseUrl ?? loadedUrl); - - return this.enqueueLinks({ ...options, urls }, crawlingContext.request, result); - }, - addRequests: result.addRequests, - pushData: result.pushData, - useState: async (defaultValue) => { - // return the old state before the browser handler was executed - // when rerunning the handler via HTTP for detection - if (oldStateCopy !== undefined) { - return oldStateCopy ?? defaultValue; // fallback to the default for `null` - } - - return this.allowStorageAccess(result.useState)(defaultValue); - }, - getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore), - }); - - await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions); - }, - this.requestHandlerTimeoutInnerMillis, - 'Request handler timed out', - ), - ); - - return { result, logs, ok: true }; - } catch (error) { - return { error, logs, ok: false }; - } - } - protected async enqueueLinks( options: SetRequired, request: RestrictedCrawlingContext['request'], @@ -708,6 +746,13 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { }, }); } + + override async teardown() { + await super.teardown(); + for (const hook of this.teardownHooks) { + await hook(); + } + } } export function createAdaptivePlaywrightRouter< diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts index 686f540c3d06..d1d9ecf54a33 100644 --- a/packages/playwright-crawler/src/internals/playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts @@ -2,31 +2,48 @@ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, - BrowserRequestHandler, GetUserDataFromRequest, - LoadedContext, + RequestHandler, RouterRoutes, } from '@crawlee/browser'; -import { BrowserCrawler, Configuration, Router } from '@crawlee/browser'; +import { BrowserCrawler, Configuration, RequestState, Router } from '@crawlee/browser'; import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool'; import type { Dictionary } from '@crawlee/types'; import ow from 'ow'; import type { LaunchOptions, Page, Response } from 'playwright'; -import type { PlaywrightLaunchContext } from './playwright-launcher'; -import { PlaywrightLauncher } from './playwright-launcher'; -import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils'; -import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils'; +import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js'; +import type { PlaywrightLaunchContext } from './playwright-launcher.js'; +import { PlaywrightLauncher } from './playwright-launcher.js'; +import type { + BlockRequestsOptions, + DirectNavigationOptions, + HandleCloudflareChallengeOptions, + InfiniteScrollOptions, + InjectFileOptions, + PlaywrightContextUtils, + SaveSnapshotOptions, +} from './utils/playwright-utils.js'; +import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js'; export interface PlaywrightCrawlingContext - extends BrowserCrawlingContext, + extends BrowserCrawlingContext, PlaywrightContextUtils {} export interface PlaywrightHook extends BrowserHook {} -export interface PlaywrightRequestHandler extends BrowserRequestHandler> {} -export type PlaywrightGotoOptions = Dictionary & Parameters[1]; +export type PlaywrightGotoOptions = Parameters[1]; -export interface PlaywrightCrawlerOptions - extends BrowserCrawlerOptions { +export interface PlaywrightCrawlerOptions< + ContextExtension = Dictionary, + ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, +> extends BrowserCrawlerOptions< + Page, + Response, + PlaywrightController, + PlaywrightCrawlingContext, + ContextExtension, + ExtendedContext, + { browserPlugins: [PlaywrightPlugin] } + > { /** * The same options as used by {@apilink launchPlaywright}. */ @@ -56,36 +73,7 @@ export interface PlaywrightCrawlerOptions * The exceptions are logged to the request using the * {@apilink Request.pushErrorMessage} function. */ - requestHandler?: PlaywrightRequestHandler; - - /** - * Function that is called to process each request. - * - * The function receives the {@apilink PlaywrightCrawlingContext} as an argument, where: - * - `request` is an instance of the {@apilink Request} object with details about the URL to open, HTTP method etc. - * - `page` is an instance of the `Playwright` - * [`Page`](https://playwright.dev/docs/api/class-page) - * - `browserController` is an instance of the - * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller), - * - `response` is an instance of the `Playwright` - * [`Response`](https://playwright.dev/docs/api/class-response), - * which is the main resource response as returned by `page.goto(request.url)`. - * - * The function must return a promise, which is then awaited by the crawler. - * - * If the function throws an exception, the crawler will try to re-crawl the - * request later, up to `option.maxRequestRetries` times. - * If all the retries fail, the crawler calls the function - * provided to the `failedRequestHandler` parameter. - * To make this work, you should **always** - * let your function throw exceptions rather than catch them. - * The exceptions are logged to the request using the - * {@apilink Request.pushErrorMessage} function. - * - * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version. - * @ignore - */ - handlePageFunction?: PlaywrightRequestHandler; + requestHandler?: RequestHandler; /** * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies @@ -187,22 +175,32 @@ export interface PlaywrightCrawlerOptions * ``` * @category Crawlers */ -export class PlaywrightCrawler extends BrowserCrawler< +export class PlaywrightCrawler< + ContextExtension = Dictionary, + ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, +> extends BrowserCrawler< + Page, + Response, + PlaywrightController, { browserPlugins: [PlaywrightPlugin] }, LaunchOptions, - PlaywrightCrawlingContext + PlaywrightCrawlingContext, + ContextExtension, + ExtendedContext > { protected static override optionsShape = { ...BrowserCrawler.optionsShape, browserPoolOptions: ow.optional.object, launcher: ow.optional.object, + ignoreIframes: ow.optional.boolean, + ignoreShadowRoots: ow.optional.boolean, }; /** * All `PlaywrightCrawler` parameters are passed via an options object. */ constructor( - private readonly options: PlaywrightCrawlerOptions = {}, + options: PlaywrightCrawlerOptions = {}, override readonly config = Configuration.getGlobalConfig(), ) { ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape)); @@ -235,12 +233,16 @@ export class PlaywrightCrawler extends BrowserCrawler< browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()]; - super({ ...browserCrawlerOptions, launchContext, browserPoolOptions }, config); - } - - protected override async _runRequestHandler(context: PlaywrightCrawlingContext) { - registerUtilsToContext(context, this.options); - await super._runRequestHandler(context); + super( + { + ...(browserCrawlerOptions as PlaywrightCrawlerOptions), + launchContext, + browserPoolOptions, + contextPipelineBuilder: () => + this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }), + }, + config, + ); } protected override async _navigationHandler( @@ -249,6 +251,60 @@ export class PlaywrightCrawler extends BrowserCrawler< ) { return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions); } + + private async enhanceContext(context: BrowserCrawlingContext) { + const waitForSelector = async (selector: string, timeoutMs = 5_000) => { + const locator = context.page.locator(selector).first(); + await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); + }; + + return { + injectFile: async (filePath: string, options?: InjectFileOptions) => + playwrightUtils.injectFile(context.page, filePath, options), + injectJQuery: async () => { + if (context.request.state === RequestState.BEFORE_NAV) { + context.log.warning( + 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.', + ); + await playwrightUtils.injectJQuery(context.page); + return; + } + await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false }); + }, + blockRequests: async (options?: BlockRequestsOptions) => + playwrightUtils.blockRequests(context.page, options), + waitForSelector, + parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => { + if (selector) { + await waitForSelector(selector, timeoutMs); + } + + return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes); + }, + infiniteScroll: async (options?: InfiniteScrollOptions) => + playwrightUtils.infiniteScroll(context.page, options), + saveSnapshot: async (options?: SaveSnapshotOptions) => + playwrightUtils.saveSnapshot(context.page, { ...options, config: this.config }), + enqueueLinksByClickingElements: async ( + options: Omit, + ) => + playwrightUtils.enqueueLinksByClickingElements({ + ...options, + page: context.page, + requestQueue: this.requestQueue!, + }), + compileScript: (scriptString: string, ctx?: Dictionary) => playwrightUtils.compileScript(scriptString, ctx), + closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page), + handleCloudflareChallenge: async (options?: HandleCloudflareChallengeOptions) => { + return playwrightUtils.handleCloudflareChallenge( + context.page, + context.request.url, + context.session, + options, + ); + }, + }; + } } /** diff --git a/packages/playwright-crawler/src/internals/playwright-launcher.ts b/packages/playwright-crawler/src/internals/playwright-launcher.ts index e9920a76a20d..a8bf13c86c50 100644 --- a/packages/playwright-crawler/src/internals/playwright-launcher.ts +++ b/packages/playwright-crawler/src/internals/playwright-launcher.ts @@ -58,13 +58,6 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext { } } -interface HandleCloudflareChallengeOptions { +export interface HandleCloudflareChallengeOptions { /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */ verbose?: boolean; /** How long should we wait after the challenge is completed for the final page to load. */ @@ -1052,52 +1045,6 @@ export interface PlaywrightContextUtils { handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise; } -export function registerUtilsToContext( - context: PlaywrightCrawlingContext, - crawlerOptions: PlaywrightCrawlerOptions, -): void { - context.injectFile = async (filePath: string, options?: InjectFileOptions) => - injectFile(context.page, filePath, options); - context.injectJQuery = async () => { - if (context.request.state === RequestState.BEFORE_NAV) { - log.warning( - 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.', - ); - await injectJQuery(context.page); - return; - } - await injectJQuery(context.page, { surviveNavigations: false }); - }; - context.blockRequests = async (options?: BlockRequestsOptions) => blockRequests(context.page, options); - context.waitForSelector = async (selector: string, timeoutMs = 5_000) => { - const locator = context.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); - }; - context.parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => { - if (selector) { - await context.waitForSelector(selector, timeoutMs); - } - - return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes); - }; - context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options); - context.saveSnapshot = async (options?: SaveSnapshotOptions) => - saveSnapshot(context.page, { ...options, config: context.crawler.config }); - context.enqueueLinksByClickingElements = async ( - options: Omit, - ) => - enqueueLinksByClickingElements({ - ...options, - page: context.page, - requestQueue: context.crawler.requestQueue!, - }); - context.compileScript = (scriptString: string, ctx?: Dictionary) => compileScript(scriptString, ctx); - context.closeCookieModals = async () => closeCookieModals(context.page); - context.handleCloudflareChallenge = async (options?: HandleCloudflareChallengeOptions) => { - return handleCloudflareChallenge(context.page, context.request.url, context.session, options); - }; -} - export { enqueueLinksByClickingElements }; /** @internal */ diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json index a84c67511053..790dc9a72c0d 100644 --- a/packages/puppeteer-crawler/package.json +++ b/packages/puppeteer-crawler/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/puppeteer", - "version": "3.15.3", + "version": "4.0.0", "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -46,25 +40,25 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "@apify/datastructures": "^2.0.0", - "@apify/log": "^2.4.0", - "@crawlee/browser": "3.15.3", - "@crawlee/browser-pool": "3.15.3", - "@crawlee/types": "3.15.3", - "@crawlee/utils": "3.15.3", - "cheerio": "1.0.0-rc.12", + "@apify/datastructures": "^2.0.3", + "@apify/log": "^2.5.18", + "@crawlee/browser": "4.0.0", + "@crawlee/browser-pool": "4.0.0", + "@crawlee/types": "4.0.0", + "@crawlee/utils": "4.0.0", + "cheerio": "^1.0.0", "devtools-protocol": "*", - "idcac-playwright": "^0.1.2", - "jquery": "^3.6.0", - "ow": "^0.28.1", - "tslib": "^2.4.0" + "idcac-playwright": "^0.1.3", + "jquery": "^3.7.1", + "ow": "^2.0.0", + "tslib": "^2.8.1" }, "peerDependencies": { "idcac-playwright": "^0.1.2", diff --git a/packages/puppeteer-crawler/src/index.ts b/packages/puppeteer-crawler/src/index.ts index ad44e8c8a00d..4d84972ba0e6 100644 --- a/packages/puppeteer-crawler/src/index.ts +++ b/packages/puppeteer-crawler/src/index.ts @@ -1,11 +1,11 @@ export * from '@crawlee/browser'; -export * from './internals/puppeteer-crawler'; -export * from './internals/puppeteer-launcher'; +export * from './internals/puppeteer-crawler.js'; +export * from './internals/puppeteer-launcher.js'; -export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception'; -export type { InterceptHandler } from './internals/utils/puppeteer_request_interception'; +export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js'; +export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js'; -export * as puppeteerUtils from './internals/utils/puppeteer_utils'; +export * as puppeteerUtils from './internals/utils/puppeteer_utils.js'; export type { BlockRequestsOptions, CompiledScriptFunction, @@ -14,7 +14,7 @@ export type { InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, -} from './internals/utils/puppeteer_utils'; +} from './internals/utils/puppeteer_utils.js'; -export * as puppeteerClickElements from './internals/enqueue-links/click-elements'; -export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements'; +export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js'; +export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js'; diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts index 2efefafde2e4..2102743526bd 100644 --- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts +++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts @@ -22,7 +22,7 @@ import type { ClickOptions, Frame, HTTPRequest as PuppeteerRequest, Page, Target import log_ from '@apify/log'; -import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception'; +import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js'; const STARTING_Z_INDEX = 2147400000; const log = log_.child({ prefix: 'Puppeteer Click Elements' }); diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts index 7580f2d51a7f..7a8da5048cb0 100644 --- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts +++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts @@ -2,31 +2,47 @@ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, - BrowserRequestHandler, GetUserDataFromRequest, - LoadedContext, RouterRoutes, } from '@crawlee/browser'; -import { BrowserCrawler, Configuration, Router } from '@crawlee/browser'; +import { BrowserCrawler, Configuration, RequestState, Router } from '@crawlee/browser'; import type { BrowserPoolOptions, PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool'; import type { Dictionary } from '@crawlee/types'; import ow from 'ow'; import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer'; -import type { PuppeteerLaunchContext } from './puppeteer-launcher'; -import { PuppeteerLauncher } from './puppeteer-launcher'; -import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils'; -import { gotoExtended, registerUtilsToContext } from './utils/puppeteer_utils'; +import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js'; +import type { PuppeteerLaunchContext } from './puppeteer-launcher.js'; +import { PuppeteerLauncher } from './puppeteer-launcher.js'; +import type { InterceptHandler } from './utils/puppeteer_request_interception.js'; +import type { + BlockRequestsOptions, + DirectNavigationOptions, + InfiniteScrollOptions, + InjectFileOptions, + PuppeteerContextUtils, + SaveSnapshotOptions, +} from './utils/puppeteer_utils.js'; +import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js'; export interface PuppeteerCrawlingContext - extends BrowserCrawlingContext, + extends BrowserCrawlingContext, PuppeteerContextUtils {} export interface PuppeteerHook extends BrowserHook {} -export interface PuppeteerRequestHandler extends BrowserRequestHandler> {} export type PuppeteerGoToOptions = Parameters[1]; -export interface PuppeteerCrawlerOptions - extends BrowserCrawlerOptions { +export interface PuppeteerCrawlerOptions< + ContextExtension = Dictionary, + ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, +> extends BrowserCrawlerOptions< + Page, + HTTPResponse, + PuppeteerController, + PuppeteerCrawlingContext, + ContextExtension, + ExtendedContext, + { browserPlugins: [PuppeteerPlugin] } + > { /** * Options used by {@apilink launchPuppeteer} to start new Puppeteer instances. */ @@ -132,10 +148,18 @@ export interface PuppeteerCrawlerOptions * ``` * @category Crawlers */ -export class PuppeteerCrawler extends BrowserCrawler< +export class PuppeteerCrawler< + ContextExtension = Dictionary, + ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, +> extends BrowserCrawler< + Page, + HTTPResponse, + PuppeteerController, { browserPlugins: [PuppeteerPlugin] }, LaunchOptions, - PuppeteerCrawlingContext + PuppeteerCrawlingContext, + ContextExtension, + ExtendedContext > { protected static override optionsShape = { ...BrowserCrawler.optionsShape, @@ -146,7 +170,7 @@ export class PuppeteerCrawler extends BrowserCrawler< * All `PuppeteerCrawler` parameters are passed via an options object. */ constructor( - private readonly options: PuppeteerCrawlerOptions = {}, + options: PuppeteerCrawlerOptions = {}, override readonly config = Configuration.getGlobalConfig(), ) { ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape)); @@ -179,12 +203,73 @@ export class PuppeteerCrawler extends BrowserCrawler< browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()]; - super({ ...browserCrawlerOptions, launchContext, proxyConfiguration, browserPoolOptions }, config); + super( + { + ...(browserCrawlerOptions as BrowserCrawlerOptions< + Page, + HTTPResponse, + PuppeteerController, + PuppeteerCrawlingContext, + ContextExtension, + ExtendedContext + >), + launchContext, + proxyConfiguration, + browserPoolOptions, + contextPipelineBuilder: () => + this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }), + }, + config, + ); } - protected override async _runRequestHandler(context: PuppeteerCrawlingContext) { - registerUtilsToContext(context, this.options); - await super._runRequestHandler(context); + private async enhanceContext(context: BrowserCrawlingContext) { + const waitForSelector = async (selector: string, timeoutMs = 5_000) => { + await context.page.waitForSelector(selector, { timeout: timeoutMs }); + }; + + return { + injectFile: async (filePath: string, options?: InjectFileOptions) => + puppeteerUtils.injectFile(context.page, filePath, options), + injectJQuery: async () => { + if (context.request.state === RequestState.BEFORE_NAV) { + context.log.warning( + 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.', + ); + await puppeteerUtils.injectJQuery(context.page); + return; + } + await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false }); + }, + waitForSelector, + parseWithCheerio: async (selector?: string, timeoutMs = 5_000) => { + if (selector) { + await waitForSelector(selector, timeoutMs); + } + + return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes); + }, + enqueueLinksByClickingElements: async ( + options: Omit, + ) => + puppeteerUtils.enqueueLinksByClickingElements({ + page: context.page, + requestQueue: this.requestQueue!, + ...options, + }), + blockRequests: async (options?: BlockRequestsOptions) => + puppeteerUtils.blockRequests(context.page, options), + compileScript: (scriptString: string, ctx?: Dictionary) => puppeteerUtils.compileScript(scriptString, ctx), + addInterceptRequestHandler: async (handler: InterceptHandler) => + puppeteerUtils.addInterceptRequestHandler(context.page, handler), + removeInterceptRequestHandler: async (handler: InterceptHandler) => + puppeteerUtils.removeInterceptRequestHandler(context.page, handler), + infiniteScroll: async (options?: InfiniteScrollOptions) => + puppeteerUtils.infiniteScroll(context.page, options), + saveSnapshot: async (options?: SaveSnapshotOptions) => + puppeteerUtils.saveSnapshot(context.page, { ...options, config: this.config }), + closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page), + }; } protected override async _navigationHandler( diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts index 8c061421f19e..a2d8b7ee855a 100644 --- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts +++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts @@ -19,10 +19,11 @@ */ import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import vm from 'node:vm'; import type { Request } from '@crawlee/browser'; -import { Configuration, KeyValueStore, RequestState, validators } from '@crawlee/browser'; +import { Configuration, KeyValueStore, validators } from '@crawlee/browser'; import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types'; import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils'; import * as cheerio from 'cheerio'; @@ -33,12 +34,12 @@ import type { HTTPRequest as PuppeteerRequest, HTTPResponse, Page, ResponseForRe import { LruCache } from '@apify/datastructures'; import log_ from '@apify/log'; -import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements'; -import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements'; -import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext } from '../puppeteer-crawler'; -import type { InterceptHandler } from './puppeteer_request_interception'; -import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception'; +import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js'; +import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js'; +import type { InterceptHandler } from './puppeteer_request_interception.js'; +import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js'; +const require = createRequire(import.meta.url); const jqueryPath = require.resolve('jquery'); const MAX_INJECT_FILE_CACHE_SIZE = 10; @@ -202,6 +203,7 @@ export async function parseWithCheerio( frames.map(async (frame) => { try { const iframe = await frame.contentFrame(); + if (iframe) { const getIframeHTML = async (): Promise => { try { @@ -958,32 +960,6 @@ export interface PuppeteerContextUtils { */ blockRequests(options?: BlockRequestsOptions): Promise; - /** - * `blockResources()` has a high impact on performance in recent versions of Puppeteer. - * Until this resolves, please use `utils.puppeteer.blockRequests()`. - * @deprecated - */ - blockResources(resourceTypes?: string[]): Promise; - - /** - * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal - * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary. - * - * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer. - * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers. - * This issue should be resolved or atleast mitigated in future iterations of this feature. - * @param cache - * Object in which responses are stored - * @param responseUrlRules - * List of rules that are used to check if the response should be cached. - * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()). - * @deprecated - */ - cacheResponses( - cache: Dictionary>, - responseUrlRules: (string | RegExp)[], - ): Promise; - /** * Compiles a Puppeteer script into an async function that may be executed at any time * by providing it with the following object: @@ -1096,60 +1072,6 @@ export interface PuppeteerContextUtils { closeCookieModals(): Promise; } -/** @internal */ -export function registerUtilsToContext( - context: PuppeteerCrawlingContext, - crawlerOptions: PuppeteerCrawlerOptions, -): void { - context.injectFile = async (filePath: string, options?: InjectFileOptions) => - injectFile(context.page, filePath, options); - context.injectJQuery = async () => { - if (context.request.state === RequestState.BEFORE_NAV) { - log.warning( - 'Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.', - ); - await injectJQuery(context.page); - return; - } - await injectJQuery(context.page, { surviveNavigations: false }); - }; - context.waitForSelector = async (selector: string, timeoutMs = 5_000) => { - await context.page.waitForSelector(selector, { timeout: timeoutMs }); - }; - context.parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => { - if (selector) { - await context.waitForSelector(selector, timeoutMs); - } - - return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes); - }; - context.enqueueLinksByClickingElements = async ( - options: Omit, - ) => - enqueueLinksByClickingElements({ - page: context.page, - requestQueue: context.crawler.requestQueue!, - ...options, - }); - context.blockRequests = async (options?: BlockRequestsOptions) => blockRequests(context.page, options); - context.blockResources = async (resourceTypes?: string[]) => blockResources(context.page, resourceTypes); - context.cacheResponses = async ( - cache: Dictionary>, - responseUrlRules: (string | RegExp)[], - ) => { - return cacheResponses(context.page, cache, responseUrlRules); - }; - context.compileScript = (scriptString: string, ctx?: Dictionary) => compileScript(scriptString, ctx); - context.addInterceptRequestHandler = async (handler: InterceptHandler) => - addInterceptRequestHandler(context.page, handler); - context.removeInterceptRequestHandler = async (handler: InterceptHandler) => - removeInterceptRequestHandler(context.page, handler); - context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options); - context.saveSnapshot = async (options?: SaveSnapshotOptions) => - saveSnapshot(context.page, { ...options, config: context.crawler.config }); - context.closeCookieModals = async () => closeCookieModals(context.page); -} - export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler }; /** @internal */ @@ -1158,8 +1080,6 @@ export const puppeteerUtils = { injectJQuery, enqueueLinksByClickingElements, blockRequests, - blockResources, - cacheResponses, compileScript, gotoExtended, addInterceptRequestHandler, diff --git a/packages/templates/package.json b/packages/templates/package.json index a94cb7384200..49634e2944b3 100644 --- a/packages/templates/package.json +++ b/packages/templates/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/templates", - "version": "3.15.3", + "version": "4.0.0", "description": "Templates for the crawlee projects", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -41,7 +35,7 @@ "scripts": { "build": "yarn clean && yarn validate && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts", "validate": "node ./scripts/validate-manifest.mjs" }, @@ -49,10 +43,6 @@ "access": "public" }, "dependencies": { - "ansi-colors": "^4.1.3", - "inquirer": "^9.0.0", - "tslib": "^2.4.0", - "yargonaut": "^1.1.4", - "yargs": "^17.5.1" + "tslib": "^2.8.1" } } diff --git a/packages/templates/templates/camoufox-ts/Dockerfile b/packages/templates/templates/camoufox-ts/Dockerfile index b86983f92d61..7b88dc88bedf 100644 --- a/packages/templates/templates/camoufox-ts/Dockerfile +++ b/packages/templates/templates/camoufox-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-playwright-chrome:20-1.50.1 AS builder +FROM apify/actor-node-playwright-chrome:22-1.50.1 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY --chown=myuser . ./ RUN npm run build # Create final image -FROM apify/actor-node-playwright-chrome:20-1.50.1 +FROM apify/actor-node-playwright-chrome:22-1.50.1 # Copy only built JS files from builder image COPY --from=builder --chown=myuser /home/myuser/dist ./dist diff --git a/packages/templates/templates/cheerio-js/Dockerfile b/packages/templates/templates/cheerio-js/Dockerfile index 4c8d11fc3f74..21f5db914654 100644 --- a/packages/templates/templates/cheerio-js/Dockerfile +++ b/packages/templates/templates/cheerio-js/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node:20 +FROM apify/actor-node:22 # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. diff --git a/packages/templates/templates/cheerio-ts/Dockerfile b/packages/templates/templates/cheerio-ts/Dockerfile index 995a3d8155c6..e15f10b68c15 100644 --- a/packages/templates/templates/cheerio-ts/Dockerfile +++ b/packages/templates/templates/cheerio-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node:20 AS builder +FROM apify/actor-node:22 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY . ./ RUN npm run build # Create final image -FROM apify/actor-node:20 +FROM apify/actor-node:22 # Copy only built JS files from builder image COPY --from=builder /usr/src/app/dist ./dist diff --git a/packages/templates/templates/empty-js/Dockerfile b/packages/templates/templates/empty-js/Dockerfile index 4c8d11fc3f74..21f5db914654 100644 --- a/packages/templates/templates/empty-js/Dockerfile +++ b/packages/templates/templates/empty-js/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node:20 +FROM apify/actor-node:22 # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. diff --git a/packages/templates/templates/empty-ts/Dockerfile b/packages/templates/templates/empty-ts/Dockerfile index 995a3d8155c6..e15f10b68c15 100644 --- a/packages/templates/templates/empty-ts/Dockerfile +++ b/packages/templates/templates/empty-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node:20 AS builder +FROM apify/actor-node:22 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY . ./ RUN npm run build # Create final image -FROM apify/actor-node:20 +FROM apify/actor-node:22 # Copy only built JS files from builder image COPY --from=builder /usr/src/app/dist ./dist diff --git a/packages/templates/templates/getting-started-js/Dockerfile b/packages/templates/templates/getting-started-js/Dockerfile index 6e804b93aadc..5ff3cde1663b 100644 --- a/packages/templates/templates/getting-started-js/Dockerfile +++ b/packages/templates/templates/getting-started-js/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-playwright-chrome:20 +FROM apify/actor-node-playwright-chrome:22 # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. diff --git a/packages/templates/templates/getting-started-ts/Dockerfile b/packages/templates/templates/getting-started-ts/Dockerfile index 1fe6784a46fc..7a033731b090 100644 --- a/packages/templates/templates/getting-started-ts/Dockerfile +++ b/packages/templates/templates/getting-started-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-playwright-chrome:20 AS builder +FROM apify/actor-node-playwright-chrome:22 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY --chown=myuser . ./ RUN npm run build # Create final image -FROM apify/actor-node-playwright-chrome:20 +FROM apify/actor-node-playwright-chrome:22 # Copy only built JS files from builder image COPY --from=builder --chown=myuser /home/myuser/dist ./dist diff --git a/packages/templates/templates/playwright-js/Dockerfile b/packages/templates/templates/playwright-js/Dockerfile index edf60c820dd0..5e6983829f86 100644 --- a/packages/templates/templates/playwright-js/Dockerfile +++ b/packages/templates/templates/playwright-js/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-playwright-chrome:20 +FROM apify/actor-node-playwright-chrome:22 # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. diff --git a/packages/templates/templates/playwright-ts/Dockerfile b/packages/templates/templates/playwright-ts/Dockerfile index 1fe6784a46fc..7a033731b090 100644 --- a/packages/templates/templates/playwright-ts/Dockerfile +++ b/packages/templates/templates/playwright-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-playwright-chrome:20 AS builder +FROM apify/actor-node-playwright-chrome:22 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY --chown=myuser . ./ RUN npm run build # Create final image -FROM apify/actor-node-playwright-chrome:20 +FROM apify/actor-node-playwright-chrome:22 # Copy only built JS files from builder image COPY --from=builder --chown=myuser /home/myuser/dist ./dist diff --git a/packages/templates/templates/puppeteer-js/Dockerfile b/packages/templates/templates/puppeteer-js/Dockerfile index fa86c423fa9c..efbbc12427e0 100644 --- a/packages/templates/templates/puppeteer-js/Dockerfile +++ b/packages/templates/templates/puppeteer-js/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-puppeteer-chrome:20 +FROM apify/actor-node-puppeteer-chrome:22 # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. diff --git a/packages/templates/templates/puppeteer-ts/Dockerfile b/packages/templates/templates/puppeteer-ts/Dockerfile index 292b6f4a156f..93d40a81b2e2 100644 --- a/packages/templates/templates/puppeteer-ts/Dockerfile +++ b/packages/templates/templates/puppeteer-ts/Dockerfile @@ -1,7 +1,7 @@ # Specify the base Docker image. You can read more about # the available images at https://crawlee.dev/docs/guides/docker-images # You can also use any other image from Docker Hub. -FROM apify/actor-node-puppeteer-chrome:20 AS builder +FROM apify/actor-node-puppeteer-chrome:22 AS builder # Copy just package.json and package-lock.json # to speed up the build using Docker layer cache. @@ -19,7 +19,7 @@ COPY --chown=myuser . ./ RUN npm run build # Create final image -FROM apify/actor-node-puppeteer-chrome:20 +FROM apify/actor-node-puppeteer-chrome:22 # Copy only built JS files from builder image COPY --from=builder --chown=myuser /home/myuser/dist ./dist diff --git a/packages/types/package.json b/packages/types/package.json index 9eee3fab90f3..a664f083db7e 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/types", - "version": "3.15.3", + "version": "4.0.0", "description": "Shared types for the crawlee projects", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -42,13 +36,14 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "publishConfig": { "access": "public" }, "dependencies": { - "tslib": "^2.4.0" + "tough-cookie": "^6.0.0", + "tslib": "^2.8.1" } } diff --git a/packages/types/src/browser.ts b/packages/types/src/browser.ts index 82f85bbc15c7..b0227df86bd9 100644 --- a/packages/types/src/browser.ts +++ b/packages/types/src/browser.ts @@ -1,4 +1,4 @@ -import type { Dictionary } from './utility-types'; +import type { Dictionary } from './utility-types.js'; export interface Cookie { /** diff --git a/packages/types/src/http-client.ts b/packages/types/src/http-client.ts new file mode 100644 index 000000000000..5ce40556ac9f --- /dev/null +++ b/packages/types/src/http-client.ts @@ -0,0 +1,90 @@ +import type { Readable } from 'node:stream'; + +import type { CookieJar } from 'tough-cookie'; + +import type { ISession } from './session.js'; +import type { AllowedHttpMethods } from './utility-types.js'; + +export type SearchParams = string | URLSearchParams | Record; + +/** + * HTTP Request as accepted by {@apilink BaseHttpClient} methods. + */ +export interface HttpRequest { + url: string | URL; + method?: AllowedHttpMethods; + headers?: Headers; + body?: Readable; + + signal?: AbortSignal; + timeout?: number; + + cookieJar?: CookieJar; + followRedirect?: boolean | ((response: any) => boolean); // TODO BC with got - specify type better in 4.0 + maxRedirects?: number; + + encoding?: BufferEncoding; + throwHttpErrors?: boolean; + + // from got-scraping Context + proxyUrl?: string; + headerGeneratorOptions?: Record; + useHeaderGenerator?: boolean; + headerGenerator?: { + getHeaders: (options: Record) => Record; + }; + insecureHTTPParser?: boolean; + sessionToken?: object; +} + +/** + * Additional options for HTTP requests that need to be handled separately before passing to {@apilink BaseHttpClient}. + */ +export interface HttpRequestOptions extends HttpRequest { + /** Search (query string) parameters to be appended to the request URL */ + searchParams?: SearchParams; + + /** A form to be sent in the HTTP request body (URL encoding will be used) */ + form?: Record; + /** Arbitrary object to be JSON-serialized and sent as the HTTP request body */ + json?: unknown; + + /** Basic HTTP Auth username */ + username?: string; + /** Basic HTTP Auth password */ + password?: string; +} + +/** + * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument. + */ +export type RedirectHandler = ( + redirectResponse: Response, + updatedRequest: { url?: string | URL; headers: Headers }, +) => void; + +export interface SendRequestOptions { + session?: ISession; + cookieJar?: CookieJar; + timeout?: number; + /** + * Overrides the proxy URL set in the `session` for this request. + * + * Note that setting this manually can interfere with session proxy rotation. + */ + proxyUrl?: string; +} + +export interface StreamOptions extends SendRequestOptions { + onRedirect?: RedirectHandler; +} + +/** + * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl. + */ +export interface BaseHttpClient { + /** + * Perform an HTTP Request and return the complete response. + */ + sendRequest(request: Request, options?: SendRequestOptions): Promise; +} diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 1130b23cb803..ef5273f56a62 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -1,3 +1,5 @@ -export * from './storages'; -export * from './utility-types'; -export * from './browser'; +export * from './storages.js'; +export * from './utility-types.js'; +export * from './browser.js'; +export * from './http-client.js'; +export * from './session.js'; diff --git a/packages/types/src/session.ts b/packages/types/src/session.ts new file mode 100644 index 000000000000..7bca512c9e32 --- /dev/null +++ b/packages/types/src/session.ts @@ -0,0 +1,199 @@ +import type { CookieJar, SerializedCookieJar } from 'tough-cookie'; + +import type { Cookie } from './browser.js'; +import type { Dictionary } from './utility-types.js'; + +/** + * The main purpose of the ProxyInfo object is to provide information + * about the current proxy connection used by the crawler for the request. + * Outside of crawlers, you can get this object by calling {@apilink ProxyConfiguration.newProxyInfo}. + * + * **Example usage:** + * + * ```javascript + * const proxyConfiguration = new ProxyConfiguration({ + * proxyUrls: ['...', '...'] // List of Proxy URLs to rotate + * }); + * + * // Getting proxyInfo object by calling class method directly + * const proxyInfo = await proxyConfiguration.newProxyInfo(); + * + * // In crawler + * const crawler = new CheerioCrawler({ + * // ... + * proxyConfiguration, + * requestHandler({ proxyInfo }) { + * // Getting used proxy URL + * const proxyUrl = proxyInfo.url; + * } + * }) + * + * ``` + */ +export interface ProxyInfo { + /** + * The URL of the proxy. + */ + url: string; + + /** + * Username for the proxy. + */ + username?: string; + + /** + * User's password for the proxy. + */ + password: string; + + /** + * Hostname of your proxy. + */ + hostname: string; + + /** + * Proxy port. + */ + port: number | string; + + /** + * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`). + */ + proxyTier?: number; + + /** + * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content. + * + * @default false + */ + ignoreTlsErrors?: boolean; +} + +/** + * Persistable {@apilink Session} state. + */ +export interface SessionState { + id: string; + cookieJar: SerializedCookieJar; + proxyInfo?: ProxyInfo; + userData: object; + errorScore: number; + maxErrorScore: number; + errorScoreDecrement: number; + usageCount: number; + maxUsageCount: number; + expiresAt: string; + createdAt: string; +} + +/** + * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions. + * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint. + * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general. + * @category Scaling + */ +export interface ISession { + readonly id: string; + userData: Dictionary; + errorScore: number; + usageCount: number; + maxErrorScore: number; + errorScoreDecrement: number; + expiresAt: Date; + createdAt: Date; + maxUsageCount: number; + cookieJar: CookieJar; + proxyInfo?: ProxyInfo; + + /** + * Indicates whether the session is blocked. + * Session is blocked once it reaches the `maxErrorScore`. + */ + isBlocked(): boolean; + + /** + * Indicates whether the session is expired. + * Session expiration is determined by the `maxAgeSecs`. + * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired. + */ + isExpired(): boolean; + + /** + * Indicates whether the session is used maximum number of times. + * Session maximum usage count can be changed by `maxUsageCount` parameter. + */ + isMaxUsageCountReached(): boolean; + + /** + * Indicates whether the session can be used for next requests. + * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached. + */ + isUsable(): boolean; + + /** + * This method should be called after a successful session usage. + * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`. + */ + markGood(): void; + + /** + * Gets session state for persistence in KeyValueStore. + * @returns Represents session internal state. + */ + getState(): SessionState; + + /** + * Marks session as blocked and emits event on the `SessionPool` + * This method should be used if the session usage was unsuccessful + * and you are sure that it is because of the session configuration and not any external matters. + * For example when server returns 403 status code. + * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method. + */ + retire(): void; + + /** + * Increases usage and error count. + * Should be used when the session has been used unsuccessfully. For example because of timeouts. + */ + markBad(): void; + + /** + * With certain status codes: `401`, `403` or `429` we can be certain + * that the target website is blocking us. This function helps to do this conveniently + * by retiring the session when such code is received. Optionally, the default status + * codes can be extended in the second parameter. + * @param statusCode HTTP status code. + * @returns Whether the session was retired. + */ + retireOnBlockedStatusCodes(statusCode: number): boolean; + + /** + * Saves cookies from an HTTP response to be used with the session. + * It expects an object with a `headers` property that's either an `Object` + * (typical Node.js responses) or a `Function` (Puppeteer Response). + * + * It then parses and saves the cookies from the `set-cookie` header, if available. + */ + setCookiesFromResponse(response: Response): void; + + /** + * Saves an array with cookie objects to be used with the session. + * The objects should be in the format that + * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls), + * but you can also use this function to set cookies manually: + * + * ``` + * [ + * { name: 'cookie1', value: 'my-cookie' }, + * { name: 'cookie2', value: 'your-cookie' } + * ] + * ``` + */ + setCookies(cookies: Cookie[], url: string): void; + + /** + * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`. + * @param url website url. Only cookies stored for this url will be returned + */ + getCookies(url: string): Cookie[]; +} diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts index ba9247626748..05c9fd048357 100644 --- a/packages/types/src/storages.ts +++ b/packages/types/src/storages.ts @@ -1,4 +1,4 @@ -import type { AllowedHttpMethods, Dictionary } from './utility-types'; +import type { AllowedHttpMethods, Dictionary } from './utility-types.js'; /** * A helper class that is used to report results from various @@ -168,6 +168,7 @@ export interface KeyValueStoreClient { delete(): Promise; listKeys(options?: KeyValueStoreClientListOptions): Promise; recordExists(key: string): Promise; + getRecordPublicUrl(key: string): Promise; getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise; setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise; deleteRecord(key: string): Promise; diff --git a/packages/types/src/utility-types.ts b/packages/types/src/utility-types.ts index 317d66d07a2b..257726831d7a 100644 --- a/packages/types/src/utility-types.ts +++ b/packages/types/src/utility-types.ts @@ -7,4 +7,22 @@ export type Constructor = new (...args: any[]) => T; /** @ignore */ export type Awaitable = T | PromiseLike; -export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH'; +export type AllowedHttpMethods = + | 'GET' + | 'HEAD' + | 'POST' + | 'PUT' + | 'DELETE' + | 'TRACE' + | 'OPTIONS' + | 'CONNECT' + | 'PATCH' + | 'get' + | 'head' + | 'post' + | 'put' + | 'delete' + | 'trace' + | 'options' + | 'connect' + | 'patch'; diff --git a/packages/utils/package.json b/packages/utils/package.json index 36d62f758976..d0cddd15d27c 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,19 +1,13 @@ { "name": "@crawlee/utils", - "version": "3.15.3", + "version": "4.0.0", "description": "A set of shared utilities that can be used by crawlers", "engines": { - "node": ">=16.0.0" + "node": ">=22.0.0" }, - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "type": "module", "exports": { - ".": { - "import": "./dist/index.mjs", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - }, + ".": "./dist/index.js", "./package.json": "./package.json" }, "keywords": [ @@ -43,24 +37,22 @@ "scripts": { "build": "yarn clean && yarn compile && yarn copy", "clean": "rimraf ./dist", - "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "compile": "tsc -p tsconfig.build.json", "copy": "tsx ../../scripts/copy.ts" }, "dependencies": { - "@apify/log": "^2.4.0", + "@apify/log": "^2.5.18", "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "3.15.3", + "@crawlee/http-client": "4.0.0", + "@crawlee/types": "4.0.0", "@types/sax": "^1.2.7", - "cheerio": "1.0.0-rc.12", - "file-type": "^20.0.0", - "got-scraping": "^4.0.3", - "ow": "^0.28.1", + "cheerio": "^1.0.0", + "domhandler": "^5.0.3", + "file-type": "^21.0.0", + "ow": "^2.0.0", "robots-parser": "^3.0.1", "sax": "^1.4.1", - "tslib": "^2.4.0", + "tslib": "^2.8.1", "whatwg-mimetype": "^4.0.0" - }, - "devDependencies": { - "@types/whatwg-mimetype": "^3.0.2" } } diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 77ff08d8832e..d6f6bf2438c3 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -1,20 +1,20 @@ -export * from './internals/blocked'; -export * from './internals/cheerio'; -export * from './internals/chunk'; -export * from './internals/extract-urls'; -export * from './internals/general'; -export * from './internals/memory-info'; -export * from './internals/debug'; -export * as social from './internals/social'; -export * from './internals/typedefs'; -export * from './internals/open_graph_parser'; -export * from './internals/gotScraping'; -export * from './internals/iterables'; -export * from './internals/robots'; -export * from './internals/sitemap'; -export * from './internals/url'; +export * from './internals/blocked.js'; +export * from './internals/cheerio.js'; +export * from './internals/chunk.js'; +export * from './internals/extract-urls.js'; +export * from './internals/general.js'; +export * from './internals/debug.js'; +export * as social from './internals/social.js'; +export * from './internals/typedefs.js'; +export * from './internals/open_graph_parser.js'; +export * from './internals/robots.js'; +export * from './internals/sitemap.js'; +export * from './internals/iterables.js'; +export * from './internals/robots.js'; +export * from './internals/sitemap.js'; +export * from './internals/url.js'; -export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info'; -export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info'; +export { getCurrentCpuTicksV2, CpuSample } from './internals/system-info/cpu-info.js'; +export { getMemoryInfo, MemoryInfo } from './internals/system-info/memory-info.js'; export { Dictionary, Awaitable, Constructor } from '@crawlee/types'; diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.ts index 2b4b79dcb8b3..7cd123b29c99 100644 --- a/packages/utils/src/internals/cheerio.ts +++ b/packages/utils/src/internals/cheerio.ts @@ -1,12 +1,12 @@ import type { Dictionary } from '@crawlee/types'; -import type { CheerioAPI, load } from 'cheerio'; +import type { CheerioAPI } from 'cheerio'; import * as cheerio from 'cheerio'; -import { tryAbsoluteURL } from './extract-urls'; +import { tryAbsoluteURL } from './extract-urls.js'; -/** @deprecated use CheerioAPI instead */ -export type CheerioRoot = ReturnType; -export type { CheerioAPI, Cheerio, Element } from 'cheerio'; +export type CheerioRoot = CheerioAPI; +export type { CheerioAPI, Cheerio } from 'cheerio'; +export type { Element } from 'domhandler'; // NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results. const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i; @@ -30,13 +30,12 @@ const BLOCK_TAGS_REGEX = * * Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML. * Optionally, to avoid duplicate parsing of HTML and thus improve performance, you can pass - * an existing Cheerio object to the function instead of the HTML text. The HTML should be parsed - * with the `decodeEntities` option set to `true`. For example: + * an existing Cheerio object to the function instead of the HTML text. * * ```javascript * import * as cheerio from 'cheerio'; * const html = 'Some text'; - * const text = htmlToText(cheerio.load(html, { decodeEntities: true })); + * const text = htmlToText(cheerio.load(html)); * ``` * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function. * @return Plain text @@ -44,10 +43,7 @@ const BLOCK_TAGS_REGEX = export function htmlToText(htmlOrCheerioElement: string | CheerioRoot): string { if (!htmlOrCheerioElement) return ''; - const $ = - typeof htmlOrCheerioElement === 'function' - ? htmlOrCheerioElement - : cheerio.load(htmlOrCheerioElement, { decodeEntities: true }); + const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : cheerio.load(htmlOrCheerioElement); let text = ''; const process = (elems: Dictionary) => { diff --git a/packages/utils/src/internals/extract-urls.ts b/packages/utils/src/internals/extract-urls.ts index 379cef9d36dd..547018da4363 100644 --- a/packages/utils/src/internals/extract-urls.ts +++ b/packages/utils/src/internals/extract-urls.ts @@ -1,7 +1,8 @@ +import { FetchHttpClient } from '@crawlee/http-client'; +import type { BaseHttpClient } from '@crawlee/types'; import ow from 'ow'; -import { URL_NO_COMMAS_REGEX } from './general'; -import { gotScraping } from './gotScraping'; +import { URL_NO_COMMAS_REGEX } from './general.js'; export interface DownloadListOfUrlsOptions { /** @@ -24,6 +25,11 @@ export interface DownloadListOfUrlsOptions { /** Allows to use a proxy for the download request. */ proxyUrl?: string; + + /** + * Custom HTTP client to use for downloading the file. + */ + httpClient?: BaseHttpClient; } /** @@ -32,15 +38,22 @@ export interface DownloadListOfUrlsOptions { */ export async function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise { ow( - options, + options as any, ow.object.exactShape({ url: ow.string.url, encoding: ow.optional.string, urlRegExp: ow.optional.regExp, proxyUrl: ow.optional.string, + httpClient: ow.optional.object, }), ); - const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl } = options; + const { + url, + encoding = 'utf8', + urlRegExp = URL_NO_COMMAS_REGEX, + proxyUrl, + httpClient = new FetchHttpClient(), + } = options; // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one. const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/); @@ -50,7 +63,11 @@ export async function downloadListOfUrls(options: DownloadListOfUrlsOptions): Pr fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`; } - const { body: string } = await gotScraping({ url: fixedUrl, encoding, proxyUrl }); + const response = await httpClient.sendRequest(new Request(fixedUrl, { method: 'GET' }), { + proxyUrl, + }); + + const string = new TextDecoder(encoding).decode(new Uint8Array(await response.arrayBuffer())); return extractUrls({ string, urlRegExp }); } @@ -73,7 +90,7 @@ export interface ExtractUrlsOptions { */ export function extractUrls(options: ExtractUrlsOptions): string[] { ow( - options, + options as any, ow.object.exactShape({ string: ow.string, urlRegExp: ow.optional.regExp, diff --git a/packages/utils/src/internals/gotScraping.ts b/packages/utils/src/internals/gotScraping.ts deleted file mode 100644 index 179ffeb2db42..000000000000 --- a/packages/utils/src/internals/gotScraping.ts +++ /dev/null @@ -1,11 +0,0 @@ -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { GotScraping } from 'got-scraping'; - -// eslint-disable-next-line import/no-mutable-exports -- Borrowing a book from NodeJS's code, we override the method with the imported one once the method is called -let gotScraping = (async (...args: Parameters) => { - ({ gotScraping } = await import('got-scraping')); - - return gotScraping(...args); -}) as GotScraping; - -export { gotScraping }; diff --git a/packages/utils/src/internals/memory-info.ts b/packages/utils/src/internals/memory-info.ts deleted file mode 100644 index 4cc8024c64a3..000000000000 --- a/packages/utils/src/internals/memory-info.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { execSync } from 'node:child_process'; -import { access, readFile } from 'node:fs/promises'; -import { freemem, totalmem } from 'node:os'; -import util from 'node:util'; - -import type { Dictionary } from '@crawlee/types'; - -import log from '@apify/log'; -// @ts-expect-error We need to add typings for @apify/ps-tree -import psTree from '@apify/ps-tree'; - -import { isDocker } from './general'; - -const MEMORY_FILE_PATHS = { - TOTAL: { - V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes', - V2: '/sys/fs/cgroup/memory.max', - }, - USED: { - V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes', - V2: '/sys/fs/cgroup/memory.current', - }, -}; - -/** - * Describes memory usage of the process. - */ -export interface MemoryInfo { - /** Total memory available in the system or container */ - totalBytes: number; - - /** Amount of free memory in the system or container */ - freeBytes: number; - - /** Amount of memory used (= totalBytes - freeBytes) */ - usedBytes: number; - - /** Amount of memory used the current Node.js process */ - mainProcessBytes: number; - - /** Amount of memory used by child processes of the current Node.js process */ - childProcessesBytes: number; -} - -/** - * Returns memory statistics of the process and the system, see {@apilink MemoryInfo}. - * - * If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits, - * otherwise it gets system memory limits. - * - * Beware that the function is quite inefficient because it spawns a new process. - * Therefore you shouldn't call it too often, like more than once per second. - */ -export async function getMemoryInfo(): Promise { - const psTreePromised = util.promisify(psTree); - - // lambda does *not* have `ps` and other command line tools - // required to extract memory usage. - const isLambdaEnvironment = process.platform === 'linux' && !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE; - - const isDockerVar = !isLambdaEnvironment && (await isDocker()); - - let mainProcessBytes = -1; - let childProcessesBytes = 0; - - if (isLambdaEnvironment) { - // reported in bytes - mainProcessBytes = process.memoryUsage().rss; - - // https://stackoverflow.com/a/55914335/129415 - const memInfo = execSync('cat /proc/meminfo').toString(); - const values = memInfo.split(/[\n: ]/).filter((val) => val.trim()); - // /proc/meminfo reports in kb, not bytes, the total used memory is reported by meminfo - // subtract memory used by the main node process in order to infer memory used by any child processes - childProcessesBytes = +values[19] * 1000 - mainProcessBytes; - } else { - // Query both root and child processes - const processes = await psTreePromised(process.pid, true); - - processes.forEach((rec: Dictionary) => { - // Skip the 'ps' or 'wmic' commands used by ps-tree to query the processes - if (rec.COMMAND === 'ps' || rec.COMMAND === 'WMIC.exe') { - return; - } - const bytes = parseInt(rec.RSS, 10); - // Obtain main process' memory separately - if (rec.PID === `${process.pid}`) { - mainProcessBytes = bytes; - return; - } - childProcessesBytes += bytes; - }); - } - - let totalBytes: number; - let usedBytes: number; - let freeBytes: number; - - if (isLambdaEnvironment) { - // memory size is defined in megabytes - totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE!, 10) * 1000000; - usedBytes = mainProcessBytes + childProcessesBytes; - freeBytes = totalBytes - usedBytes; - - log.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`); - } else if (isDockerVar) { - // When running inside Docker container, use container memory limits - - // Check whether cgroups V1 or V2 is used - let cgroupsVersion: keyof typeof MEMORY_FILE_PATHS.TOTAL = 'V1'; - try { - // If this directory does not exists, assume docker is using cgroups V2 - await access('/sys/fs/cgroup/memory/'); - } catch { - cgroupsVersion = 'V2'; - } - - try { - let [totalBytesStr, usedBytesStr] = await Promise.all([ - readFile(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], 'utf8'), - readFile(MEMORY_FILE_PATHS.USED[cgroupsVersion], 'utf8'), - ]); - - // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code. - totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, ''); - usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, ''); - - // Cgroups V2 contains 'max' string if memory is not limited - // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max") - if (totalBytesStr === 'max') { - totalBytes = totalmem(); - // Cgroups V1 is set to number related to platform and page size if memory is not limited - // See https://unix.stackexchange.com/q/420906 - } else { - totalBytes = parseInt(totalBytesStr, 10); - const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER; - if (containerRunsWithUnlimitedMemory) totalBytes = totalmem(); - } - usedBytes = parseInt(usedBytesStr, 10); - freeBytes = totalBytes - usedBytes; - } catch (err) { - // log.deprecated logs a warning only once - log.deprecated( - 'Your environment is Docker, but your system does not support memory cgroups. ' + - "If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" + - `Cause: ${(err as Error).message}`, - ); - totalBytes = totalmem(); - freeBytes = freemem(); - usedBytes = totalBytes - freeBytes; - } - } else { - totalBytes = totalmem(); - freeBytes = freemem(); - usedBytes = totalBytes - freeBytes; - } - - return { - totalBytes, - freeBytes, - usedBytes, - mainProcessBytes, - childProcessesBytes, - }; -} diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts index ce54f86186e7..21fc911cfb34 100644 --- a/packages/utils/src/internals/robots.ts +++ b/packages/utils/src/internals/robots.ts @@ -1,12 +1,9 @@ -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood -import type { HTTPError as HTTPErrorClass } from 'got-scraping'; +import { FetchHttpClient } from '@crawlee/http-client'; +import type { BaseHttpClient } from '@crawlee/types'; import type { Robot } from 'robots-parser'; import robotsParser from 'robots-parser'; -import { gotScraping } from './gotScraping'; -import { Sitemap } from './sitemap'; - -let HTTPError: typeof HTTPErrorClass; +import { Sitemap } from './sitemap.js'; /** * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt). @@ -37,12 +34,15 @@ export class RobotsTxtFile { * @param url the URL to fetch robots.txt for * @param [proxyUrl] a proxy to be used for fetching the robots.txt file */ - static async find(url: string, proxyUrl?: string): Promise { + static async find( + url: string, + options?: { proxyUrl?: string; httpClient?: BaseHttpClient }, + ): Promise { const robotsTxtFileUrl = new URL(url); robotsTxtFileUrl.pathname = '/robots.txt'; robotsTxtFileUrl.search = ''; - return RobotsTxtFile.load(robotsTxtFileUrl.toString(), proxyUrl); + return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options); } /** @@ -52,39 +52,40 @@ export class RobotsTxtFile { * @param [proxyUrl] a proxy to be used for fetching the robots.txt file */ static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile { + // @ts-ignore return new RobotsTxtFile(robotsParser(url, content), proxyUrl); } - protected static async load(url: string, proxyUrl?: string): Promise { - if (!HTTPError) { - HTTPError = (await import('got-scraping')).HTTPError; - } + protected static async load( + url: string, + options?: { proxyUrl?: string; httpClient?: BaseHttpClient }, + ): Promise { + const { proxyUrl, httpClient = new FetchHttpClient() } = options || {}; - try { - const response = await gotScraping({ - url, - proxyUrl, - method: 'GET', - responseType: 'text', - }); + const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), { + proxyUrl, + }); + + if (response.status < 200 || response.status >= 300) { + throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`); + } - return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl); - } catch (e) { - if (e instanceof HTTPError && e.response.statusCode === 404) { - return new RobotsTxtFile( - { - isAllowed() { - return true; - }, - getSitemaps() { - return []; - }, + if (response.status === 404) { + return new RobotsTxtFile( + { + isAllowed() { + return true; + }, + getSitemaps() { + return []; }, - proxyUrl, - ); - } - throw e; + }, + proxyUrl, + ); } + + // @ts-ignore + return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl); } /** diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts index aab562e56d8b..25e1776467be 100644 --- a/packages/utils/src/internals/sitemap.ts +++ b/packages/utils/src/internals/sitemap.ts @@ -4,8 +4,9 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream'; import { StringDecoder } from 'node:string_decoder'; import { createGunzip } from 'node:zlib'; -// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types -import type { Delays } from 'got-scraping'; +import { FetchHttpClient } from '@crawlee/http-client'; +import type { BaseHttpClient } from '@crawlee/types'; +import { fileTypeStream } from 'file-type'; import sax from 'sax'; import MIMEType from 'whatwg-mimetype'; @@ -183,14 +184,18 @@ export interface ParseSitemapOptions { */ sitemapRetries?: number; /** - * Network timeouts for sitemap fetching. See [Got documentation](https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md) for more details. + * Timeout settings for network requests when fetching sitemaps. By default this is `30000` milliseconds (30 seconds). */ - networkTimeouts?: Delays; + timeoutMillis?: number; /** * If true, the parser will log a warning if it fails to fetch a sitemap due to a network error * @default true */ reportNetworkErrors?: boolean; + /** + * Custom HTTP client to be used for fetching sitemaps. + */ + httpClient?: BaseHttpClient; } export async function* parseSitemap( @@ -198,13 +203,12 @@ export async function* parseSitemap( proxyUrl?: string, options?: T, ): AsyncIterable { - const { gotScraping } = await import('got-scraping'); - const { fileTypeStream } = await import('file-type'); const { + httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, - networkTimeouts, + timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {}; @@ -250,28 +254,34 @@ export async function* parseSitemap( while (retriesLeft-- > 0) { try { - const sitemapStream = await new Promise>( - (resolve, reject) => { - const request = gotScraping.stream({ - url: sitemapUrl, - proxyUrl, + let sitemapResponse: Response | null; + + try { + sitemapResponse = await httpClient.sendRequest( + new Request(sitemapUrl, { method: 'GET', - timeout: networkTimeouts, headers: { accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8', }, - }); - request.on('response', () => resolve(request)); - request.on('error', reject); - }, - ); + }), + { + proxyUrl, + timeout, + }, + ); + } catch (error: any) { + sitemapResponse = null; + } let error: { error: Error; type: 'fetch' | 'parser' } | null = null; - if (sitemapStream.response!.statusCode >= 200 && sitemapStream.response!.statusCode < 300) { - let contentType = sitemapStream.response!.headers['content-type']; + if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) { + let contentType = sitemapResponse.headers.get('content-type'); - const streamWithType = await fileTypeStream(sitemapStream); + if (sitemapResponse.body === null) { + break; + } + const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body as any)); if (streamWithType.fileType !== undefined) { contentType = streamWithType.fileType.mime; } @@ -293,7 +303,7 @@ export async function* parseSitemap( items = pipeline( streamWithType, isGzipped ? createGunzip() : new PassThrough(), - createParser(contentType, sitemapUrl), + createParser(contentType ?? undefined, sitemapUrl), (e) => { if (e !== undefined && e !== null) { error = { type: 'parser', error: e }; @@ -304,7 +314,7 @@ export async function* parseSitemap( error = { type: 'fetch', error: new Error( - `Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapStream.response!.statusCode}`, + `Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`, ), }; } @@ -377,7 +387,11 @@ export class Sitemap { * @param url The domain URL to fetch the sitemap for. * @param proxyUrl A proxy to be used for fetching the sitemap file. */ - static async tryCommonNames(url: string, proxyUrl?: string): Promise { + static async tryCommonNames( + url: string, + proxyUrl?: string, + parseSitemapOptions?: ParseSitemapOptions, + ): Promise { const sitemapUrls: string[] = []; const sitemapUrl = new URL(url); @@ -389,7 +403,7 @@ export class Sitemap { sitemapUrl.pathname = '/sitemap.txt'; sitemapUrls.push(sitemapUrl.toString()); - return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false }); + return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions }); } /** @@ -414,8 +428,12 @@ export class Sitemap { * @param content XML sitemap content * @param proxyUrl URL of a proxy to be used for fetching sitemap contents */ - static async fromXmlString(content: string, proxyUrl?: string): Promise { - return await this.parse([{ type: 'raw', content }], proxyUrl); + static async fromXmlString( + content: string, + proxyUrl?: string, + parseSitemapOptions?: ParseSitemapOptions, + ): Promise { + return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions); } protected static async parse( diff --git a/packages/utils/src/internals/social.ts b/packages/utils/src/internals/social.ts index 2c4a6a179a8a..f6a9a4957d9a 100644 --- a/packages/utils/src/internals/social.ts +++ b/packages/utils/src/internals/social.ts @@ -1,6 +1,6 @@ import * as cheerio from 'cheerio'; -import { htmlToText } from './cheerio'; +import { htmlToText } from './cheerio.js'; // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/ const EMAIL_REGEX_STRING = @@ -675,7 +675,7 @@ export function parseHandlesFromHtml(html: string, data: Record if ((typeof html as unknown) !== 'string') return result; - const $ = cheerio.load(html, { decodeEntities: true }); + const $ = cheerio.load(html, { xml: { decodeEntities: true } }); if (data) data.$ = $; const text = htmlToText($); diff --git a/packages/utils/src/internals/systemInfoV2/cpu-info.ts b/packages/utils/src/internals/system-info/cpu-info.ts similarity index 99% rename from packages/utils/src/internals/systemInfoV2/cpu-info.ts rename to packages/utils/src/internals/system-info/cpu-info.ts index 94f55d9b8e00..95cd1359507e 100644 --- a/packages/utils/src/internals/systemInfoV2/cpu-info.ts +++ b/packages/utils/src/internals/system-info/cpu-info.ts @@ -4,7 +4,7 @@ import os from 'node:os'; import log from '@apify/log'; -import { getCgroupsVersion } from '../general'; +import { getCgroupsVersion } from '../general.js'; const CPU_FILE_PATHS = { STAT: { diff --git a/packages/utils/src/internals/systemInfoV2/memory-info.ts b/packages/utils/src/internals/system-info/memory-info.ts similarity index 96% rename from packages/utils/src/internals/systemInfoV2/memory-info.ts rename to packages/utils/src/internals/system-info/memory-info.ts index 96c005bf2f65..855fde4591f9 100644 --- a/packages/utils/src/internals/systemInfoV2/memory-info.ts +++ b/packages/utils/src/internals/system-info/memory-info.ts @@ -4,8 +4,8 @@ import { freemem, totalmem } from 'node:os'; import log from '@apify/log'; -import { getCgroupsVersion, isLambda } from '../general'; -import { psTree } from './ps-tree'; +import { getCgroupsVersion, isLambda } from '../general.js'; +import { psTree } from './ps-tree.js'; const MEMORY_FILE_PATHS = { TOTAL: { @@ -49,7 +49,7 @@ export interface MemoryInfo { * @returns An object containing the free and used memory metrics. * @internal */ -export async function getMemoryInfoV2(containerized = false): Promise { +export async function getMemoryInfo(containerized = false): Promise { let mainProcessBytes = -1; let childProcessesBytes = 0; diff --git a/packages/utils/src/internals/systemInfoV2/ps-tree.ts b/packages/utils/src/internals/system-info/ps-tree.ts similarity index 100% rename from packages/utils/src/internals/systemInfoV2/ps-tree.ts rename to packages/utils/src/internals/system-info/ps-tree.ts diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.ts index c13d674ea478..5d7116d55b3c 100644 --- a/packages/utils/src/internals/url.ts +++ b/packages/utils/src/internals/url.ts @@ -1,4 +1,4 @@ -export type SearchParams = string | URLSearchParams | Record; +import type { SearchParams } from '@crawlee/types'; /** * Appends search (query string) parameters to a URL, replacing the original value (if any). diff --git a/packages/utils/test/mock-http-client.ts b/packages/utils/test/mock-http-client.ts new file mode 100644 index 000000000000..14673d5604b1 --- /dev/null +++ b/packages/utils/test/mock-http-client.ts @@ -0,0 +1,13 @@ +import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/types'; + +export class FetchHttpClient implements BaseHttpClient { + async sendRequest(request: Request, options?: SendRequestOptions): Promise { + const signal = AbortSignal.timeout(options?.timeout ?? 30000); + const response = await fetch(request, { signal }); + return response; + } + + async stream(request: Request, options?: StreamOptions): Promise { + return this.sendRequest(request, options); + } +} diff --git a/packages/utils/test/non-error-objects-working.test.ts b/packages/utils/test/non-error-objects-working.test.ts index c7adfbfbb511..47e28a1e9b69 100644 --- a/packages/utils/test/non-error-objects-working.test.ts +++ b/packages/utils/test/non-error-objects-working.test.ts @@ -1,4 +1,4 @@ -import { ErrorTracker } from '../../core/src/crawlers/error_tracker'; +import { ErrorTracker } from '../../core/src/crawlers/error_tracker.js'; describe('ErrorTracker', () => { test('processing a non-error error should not crash', () => { diff --git a/packages/utils/test/robots.test.ts b/packages/utils/test/robots.test.ts index 7c775ff03582..eeeb1a5df31c 100644 --- a/packages/utils/test/robots.test.ts +++ b/packages/utils/test/robots.test.ts @@ -1,7 +1,10 @@ import nock from 'nock'; import { beforeEach, describe, expect, it } from 'vitest'; -import { RobotsTxtFile } from '../src/internals/robots'; +import { RobotsTxtFile } from '../src/internals/robots.js'; +import { FetchHttpClient } from './mock-http-client.js'; + +const httpClient = new FetchHttpClient(); describe('RobotsTxtFile', () => { beforeEach(() => { @@ -37,12 +40,12 @@ describe('RobotsTxtFile', () => { }); it('generates the correct robots.txt URL', async () => { - const robots = await RobotsTxtFile.find('http://not-exists.com/nested/index.html'); + const robots = await RobotsTxtFile.find('http://not-exists.com/nested/index.html', { httpClient }); expect(robots.getSitemaps()).not.toHaveLength(0); }); it('parses allow/deny directives from robots.txt', async () => { - const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt'); + const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt', { httpClient }); console.log(robots.isAllowed('https://crawlee.dev')); expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true); expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true); @@ -50,7 +53,7 @@ describe('RobotsTxtFile', () => { }); it('extracts sitemap urls', async () => { - const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt'); + const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt', { httpClient }); expect(robots.getSitemaps()).toEqual([ 'http://not-exists.com/sitemap_1.xml', 'http://not-exists.com/sitemap_2.xml', diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts index 5b3e53e7a4ff..0e30f49acca0 100644 --- a/packages/utils/test/sitemap.test.ts +++ b/packages/utils/test/sitemap.test.ts @@ -3,8 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import log from '@apify/log'; -import type { SitemapUrl } from '../src/internals/sitemap'; -import { parseSitemap, Sitemap } from '../src/internals/sitemap'; +import type { SitemapUrl } from '../src/internals/sitemap.js'; +import { parseSitemap, Sitemap } from '../src/internals/sitemap.js'; +import { FetchHttpClient } from './mock-http-client.js'; describe('Sitemap', () => { beforeEach(() => { @@ -233,7 +234,9 @@ describe('Sitemap', () => { }); it('extracts urls from sitemaps', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_child.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_child.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/', @@ -248,7 +251,11 @@ describe('Sitemap', () => { it('extracts metadata from sitemaps', async () => { const items: SitemapUrl[] = []; - for await (const item of parseSitemap([{ type: 'url', url: 'http://not-exists.com/sitemap_child.xml' }])) { + for await (const item of parseSitemap( + [{ type: 'url', url: 'http://not-exists.com/sitemap_child.xml' }], + undefined, + { httpClient: new FetchHttpClient() }, + )) { items.push(item); } @@ -264,7 +271,9 @@ describe('Sitemap', () => { }); it('extracts urls from gzipped sitemaps', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_child.xml.gz'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_child.xml.gz', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/', @@ -277,12 +286,16 @@ describe('Sitemap', () => { }); it('identifies incorrect gzipped sitemaps as malformed', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/invalid_sitemap_child.xml.gz'); + const sitemap = await Sitemap.load('http://not-exists.com/invalid_sitemap_child.xml.gz', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual(new Set([])); }); it('follows links in sitemap indexes', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/', @@ -295,17 +308,23 @@ describe('Sitemap', () => { }); it('does not break on invalid xml', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/not_actual_xml.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/not_actual_xml.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(sitemap.urls).toEqual([]); }); it('handles CDATA in loc tags', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_cdata.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_cdata.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual(new Set(['http://not-exists.com/catalog'])); }); it('autodetects sitemaps', async () => { - const sitemap = await Sitemap.tryCommonNames('http://not-exists.com/arbitrary_url?search=xyz'); + const sitemap = await Sitemap.tryCommonNames('http://not-exists.com/arbitrary_url?search=xyz', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/catalog?item=80&desc=vacation_turkey', @@ -319,14 +338,18 @@ describe('Sitemap', () => { it('keeps quiet if autodetection does not find anything', async () => { const spy = vi.spyOn(log, 'warning'); - const sitemap = await Sitemap.tryCommonNames('http://not-exists-2.com/arbitrary_url?search=xyz'); + const sitemap = await Sitemap.tryCommonNames('http://not-exists-2.com/arbitrary_url?search=xyz', undefined, { + httpClient: new FetchHttpClient(), + }); expect(sitemap.urls).toHaveLength(0); expect(spy).not.toHaveBeenCalled(); }); it('handles sitemap.txt correctly', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap.txt'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap.txt', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/catalog?item=78&desc=vacation_crete', @@ -336,14 +359,20 @@ describe('Sitemap', () => { }); it('handles pretty-printed XML correctly', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_pretty.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_pretty.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual(new Set(['http://not-exists.com/catalog?item=80&desc=vacation_turkey'])); }); it('extracts metadata from pretty-printed XML', async () => { const items: SitemapUrl[] = []; - for await (const item of parseSitemap([{ type: 'url', url: 'http://not-exists.com/sitemap_pretty.xml' }])) { + for await (const item of parseSitemap( + [{ type: 'url', url: 'http://not-exists.com/sitemap_pretty.xml' }], + undefined, + { httpClient: new FetchHttpClient() }, + )) { items.push(item); } @@ -359,7 +388,9 @@ describe('Sitemap', () => { }); it('handles pretty-printed nested sitemaps XML correctly', async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent_pretty.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent_pretty.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ 'http://not-exists.com/', @@ -411,6 +442,8 @@ describe('Sitemap', () => { '', '', ].join('\n'), + undefined, + { httpClient: new FetchHttpClient() }, ); expect(new Set(sitemap.urls)).toEqual( @@ -425,7 +458,9 @@ describe('Sitemap', () => { }); it("loads XML sitemap even though it's gzipped according to file extension", async () => { - const sitemap = await Sitemap.load('http://not-exists.com/non_gzipped_sitemap.xml.gz'); + const sitemap = await Sitemap.load('http://not-exists.com/non_gzipped_sitemap.xml.gz', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ @@ -436,7 +471,9 @@ describe('Sitemap', () => { }); it("loads gzipped sitemap even though it's not gzipped according to file extension", async () => { - const sitemap = await Sitemap.load('http://not-exists.com/sneakily_gzipped_sitemap.xml'); + const sitemap = await Sitemap.load('http://not-exists.com/sneakily_gzipped_sitemap.xml', undefined, { + httpClient: new FetchHttpClient(), + }); expect(new Set(sitemap.urls)).toEqual( new Set([ diff --git a/renovate.json b/renovate.json index 3265e8a37601..77e1c61e18a2 100644 --- a/renovate.json +++ b/renovate.json @@ -26,5 +26,5 @@ "schedule": ["every weekday"], "minimumReleaseAge": "1 day", "internalChecksFilter": "strict", - "ignoreDeps": ["crawlee", "cheerio", "yarn"] + "ignoreDeps": ["crawlee"] } diff --git a/scripts/copy.ts b/scripts/copy.ts index 2c402438646d..e1e564febd0b 100644 --- a/scripts/copy.ts +++ b/scripts/copy.ts @@ -1,8 +1,11 @@ -/* eslint-disable import/no-dynamic-require,global-require */ +/* eslint-disable import/no-dynamic-require */ import { execSync } from 'node:child_process'; import { copyFileSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; import { resolve } from 'node:path'; +const require = createRequire(import.meta.url); + const options = process.argv.slice(2).reduce((args, arg) => { const [key, value] = arg.split('='); args[key.substring(2)] = value ?? true; @@ -31,11 +34,31 @@ function getRootVersion(bump = true): string { return rootVersion; } - rootVersion = require(resolve(root, './lerna.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1'); + const pkg = require(resolve(root, './lerna.json')); + rootVersion = pkg.version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1'); if (bump) { const parts = rootVersion.split('.'); - parts[2] = `${+parts[2] + 1}`; + const inc = bump ? 1 : 0; + const canary = String(options.canary).toLowerCase(); + + switch (canary) { + case 'major': { + parts[0] = `${+parts[0] + inc}`; + parts[1] = '0'; + parts[2] = '0'; + break; + } + case 'minor': { + parts[1] = `${+parts[0] + inc}`; + parts[2] = '0'; + break; + } + case 'patch': + default: + parts[2] = `${+parts[2] + inc}`; + } + rootVersion = parts.join('.'); } @@ -77,7 +100,7 @@ function getNextVersion() { // as we publish only the dist folder, we need to copy some meta files inside (readme/license/package.json) // also changes paths inside the copied `package.json` (`dist/index.js` -> `index.js`) -const root = resolve(__dirname, '..'); +const root = resolve(import.meta.dirname, '..'); const target = resolve(process.cwd(), 'dist'); const pkgPath = resolve(process.cwd(), 'package.json'); diff --git a/test/browser-pool/anonymize-proxy-sugar.test.ts b/test/browser-pool/anonymize-proxy-sugar.test.ts index c4a08cd96574..8e16110ebeab 100644 --- a/test/browser-pool/anonymize-proxy-sugar.test.ts +++ b/test/browser-pool/anonymize-proxy-sugar.test.ts @@ -1,7 +1,7 @@ import { anonymizeProxy } from 'proxy-chain'; import { vi } from 'vitest'; -import { anonymizeProxySugar } from '../../packages/browser-pool/src/anonymize-proxy'; +import { anonymizeProxySugar } from '../../packages/browser-pool/src/anonymize-proxy.js'; describe('anonymizeProxySugar', () => { // Mock the anonymizeProxy function from proxy-chain @@ -20,13 +20,10 @@ describe('anonymizeProxySugar', () => { ['http://username:password@proxy:1000/', 'http://username:password@proxy:1000'], ['socks://username:password@proxy:1000', 'socks://username:password@proxy:1000'], ['socks://username:password@proxy:1000/', 'socks://username:password@proxy:1000'], - ])( - 'should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s', - async (input, expectedOutput) => { - const [anonymized] = await anonymizeProxySugar(input); + ])('should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s', async (input, expectedOutput) => { + const [anonymized] = await anonymizeProxySugar(input); - expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput); - expect(anonymized).toBeTypeOf('string'); - }, - ); + expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput); + expect(anonymized).toBeTypeOf('string'); + }); }); diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.ts index 5ceebafa760b..2d03e0926658 100644 --- a/test/browser-pool/browser-plugins/plugins.test.ts +++ b/test/browser-pool/browser-plugins/plugins.test.ts @@ -16,9 +16,9 @@ import playwright from 'playwright'; import type { Server as ProxyChainServer } from 'proxy-chain'; import type { Browser } from 'puppeteer'; import puppeteer from 'puppeteer'; -import { runExampleComServer } from 'test/shared/_helper'; +import { runExampleComServer } from 'test/shared/_helper.js'; -import { createProxyServer } from './create-proxy-server'; +import { createProxyServer } from './create-proxy-server.js'; vitest.setConfig({ testTimeout: 120_000 }); @@ -159,7 +159,7 @@ const runPluginTest = < expect(false).toBe(true); } catch (error: any) { expect(error.message).toBe( - 'A new page can be created with provided context only when using incognito pages or experimental containers.', + 'A new page can be created with provided context only when using incognito pages.', ); } }); diff --git a/test/browser-pool/browser-pool.test.ts b/test/browser-pool/browser-pool.test.ts index c20cf65bdb32..3b68624e7f00 100644 --- a/test/browser-pool/browser-pool.test.ts +++ b/test/browser-pool/browser-pool.test.ts @@ -11,13 +11,13 @@ import puppeteer from 'puppeteer'; import { addTimeoutToPromise } from '@apify/timeout'; -import type { BrowserController } from '../../packages/browser-pool/src/abstract-classes/browser-controller'; -import { BrowserPool } from '../../packages/browser-pool/src/browser-pool'; -import { BROWSER_POOL_EVENTS } from '../../packages/browser-pool/src/events'; -import { BrowserName, OperatingSystemsName } from '../../packages/browser-pool/src/fingerprinting/types'; -import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin'; -import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin'; -import { createProxyServer } from './browser-plugins/create-proxy-server'; +import type { BrowserController } from '../../packages/browser-pool/src/abstract-classes/browser-controller.js'; +import { BrowserPool } from '../../packages/browser-pool/src/browser-pool.js'; +import { BROWSER_POOL_EVENTS } from '../../packages/browser-pool/src/events.js'; +import { BrowserName, OperatingSystemsName } from '../../packages/browser-pool/src/fingerprinting/types.js'; +import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin.js'; +import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin.js'; +import { createProxyServer } from './browser-plugins/create-proxy-server.js'; const fingerprintingMatrix: [string, PlaywrightPlugin | PuppeteerPlugin][] = [ [ @@ -535,7 +535,7 @@ describe.each([ }); test('should hide webdriver', async () => { - await page.goto(`file://${__dirname}/test.html`); + await page.goto(`file://${import.meta.dirname}/test.html`); const webdriver = await page.evaluate(() => { return navigator.webdriver; }); @@ -566,7 +566,7 @@ describe.each([ }); test('should override fingerprint', async () => { - await page.goto(`file://${__dirname}/test.html`); + await page.goto(`file://${import.meta.dirname}/test.html`); // @ts-expect-error mistypings const browserController = browserPoolWithFP.getBrowserControllerByPage(page); @@ -585,7 +585,7 @@ describe.each([ }); test('should hide webdriver', async () => { - await page.goto(`file://${__dirname}/test.html`); + await page.goto(`file://${import.meta.dirname}/test.html`); const webdriver = await page.evaluate(() => { return navigator.webdriver; }); diff --git a/test/browser-pool/index.test.ts b/test/browser-pool/index.test.ts index fa8d93d4f996..d6a121015636 100644 --- a/test/browser-pool/index.test.ts +++ b/test/browser-pool/index.test.ts @@ -1,8 +1,8 @@ import * as modules from '@crawlee/browser-pool'; -import { BrowserPool } from '../../packages/browser-pool/src/browser-pool'; -import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin'; -import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin'; +import { BrowserPool } from '../../packages/browser-pool/src/browser-pool.js'; +import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin.js'; +import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin.js'; describe('Exports', () => { test('Modules', () => { diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.ts index c6e019b92df5..27f24ba1dc87 100644 --- a/test/core/autoscaling/snapshotter.test.ts +++ b/test/core/autoscaling/snapshotter.test.ts @@ -214,7 +214,7 @@ describe('Snapshotter', () => { mainProcessBytes: toBytes(1000), childProcessesBytes: toBytes(1000), } as MemoryInfo; - vitest.spyOn(utils, 'getMemoryInfoV2').mockResolvedValue(memoryData); + vitest.spyOn(utils, 'getMemoryInfo').mockResolvedValue(memoryData); const config = new Configuration({ availableMemoryRatio: 1 }); const snapshotter = new Snapshotter({ config, maxUsedMemoryRatio: 0.5 }); // do not initialize the event intervals as we will fire them manually @@ -245,7 +245,7 @@ describe('Snapshotter', () => { }); test('correctly logs critical memory overload', async () => { - vitest.spyOn(utils, 'getMemoryInfoV2').mockResolvedValueOnce({ totalBytes: toBytes(10000) } as MemoryInfo); + vitest.spyOn(utils, 'getMemoryInfo').mockResolvedValueOnce({ totalBytes: toBytes(10000) } as MemoryInfo); const config = new Configuration({ availableMemoryRatio: 1 }); const snapshotter = new Snapshotter({ config, maxUsedMemoryRatio: 0.5 }); await snapshotter.start(); diff --git a/test/core/browser_launchers/playwright_launcher.test.ts b/test/core/browser_launchers/playwright_launcher.test.ts index 6e66f060dfa5..c9383809d6aa 100644 --- a/test/core/browser_launchers/playwright_launcher.test.ts +++ b/test/core/browser_launchers/playwright_launcher.test.ts @@ -11,9 +11,8 @@ import basicAuthParser from 'basic-auth-parser'; import type { Browser, BrowserType } from 'playwright'; // @ts-expect-error no types import portastic from 'portastic'; -// @ts-expect-error no types -import proxy from 'proxy'; -import { runExampleComServer } from 'test/shared/_helper'; +import { createProxy } from 'proxy'; +import { runExampleComServer } from 'test/shared/_helper.js'; let prevEnvHeadless: boolean; let proxyServer: Server; @@ -41,24 +40,23 @@ beforeAll(async () => { // Setup proxy authorization // @ts-expect-error - httpServer.authenticate = function (req, fn) { + httpServer.authenticate = function (req) { // parse the "Proxy-Authorization" header const auth = req.headers['proxy-authorization']; if (!auth) { // optimization: don't invoke the child process if no // "Proxy-Authorization" header was given - fn(null, false); - return; + return false; } const parsed = basicAuthParser(auth); const isEqual = JSON.stringify(parsed) === JSON.stringify(proxyAuth); if (isEqual) wasProxyCalled = true; - fn(null, isEqual); + return isEqual; }; httpServer.on('error', reject); - proxyServer = proxy(httpServer); + proxyServer = createProxy(httpServer); proxyServer.listen(ports[0], () => { proxyPort = (proxyServer.address() as AddressInfo).port; resolve(); @@ -274,7 +272,7 @@ describe('launchPlaywright()', () => { }); test('supports userDataDir', async () => { - const userDataDir = path.join(__dirname, 'userDataPlaywright'); + const userDataDir = path.join(import.meta.dirname, 'userDataPlaywright'); let browser; try { diff --git a/test/core/browser_launchers/puppeteer_launcher.test.ts b/test/core/browser_launchers/puppeteer_launcher.test.ts index 762248941066..0963cd62de6b 100644 --- a/test/core/browser_launchers/puppeteer_launcher.test.ts +++ b/test/core/browser_launchers/puppeteer_launcher.test.ts @@ -11,11 +11,10 @@ import type { Dictionary } from '@crawlee/utils'; import basicAuthParser from 'basic-auth-parser'; // @ts-expect-error no types import portastic from 'portastic'; -// @ts-expect-error no types -import proxy from 'proxy'; +import { createProxy } from 'proxy'; import type { Browser, Page } from 'puppeteer'; -import { runExampleComServer } from '../../shared/_helper'; +import { runExampleComServer } from '../../shared/_helper.js'; let prevEnvHeadless: string | undefined; let proxyServer: Server; @@ -64,7 +63,7 @@ beforeAll(() => { httpServer.on('error', reject); - proxyServer = proxy(httpServer); + proxyServer = createProxy(httpServer); proxyServer.listen(ports[0], () => { proxyPort = (proxyServer.address() as AddressInfo).port; resolve(); @@ -287,7 +286,7 @@ describe('launchPuppeteer()', () => { }); test('supports userDataDir', async () => { - const userDataDir = path.join(__dirname, 'userDataPuppeteer'); + const userDataDir = path.join(import.meta.dirname, 'userDataPuppeteer'); let browser; try { diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts index 589fd88ab4e1..38468681ef25 100644 --- a/test/core/crawlers/adaptive_playwright_crawler.test.ts +++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts @@ -11,8 +11,8 @@ import type { import { AdaptivePlaywrightCrawler, RenderingTypePredictor, RequestList } from '@crawlee/playwright'; import { sleep } from 'crawlee'; import express from 'express'; -import { startExpressAppPromise } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { startExpressAppPromise } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; describe('AdaptivePlaywrightCrawler', () => { // Set up an express server that will serve test pages @@ -234,34 +234,34 @@ describe('AdaptivePlaywrightCrawler', () => { expect(resultChecker).toHaveBeenCalledTimes(1); }); - test.each([['static'], ['clientOnly']] as const)( - 'crawlingContext.addRequests() should add requests correctly (%s)', - async (renderingType) => { - const renderingTypePredictor = makeRiggedRenderingTypePredictor({ - detectionProbabilityRecommendation: 0, - renderingType, - }); - const url = new URL(`http://${HOSTNAME}:${port}`).toString(); + test.each([ + ['static'], + ['clientOnly'], + ] as const)('crawlingContext.addRequests() should add requests correctly (%s)', async (renderingType) => { + const renderingTypePredictor = makeRiggedRenderingTypePredictor({ + detectionProbabilityRecommendation: 0, + renderingType, + }); + const url = new URL(`http://${HOSTNAME}:${port}`).toString(); - let requestContext: LoadedContext | undefined; - const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => { - const isStartUrl = context.request.url === url; + let requestContext: LoadedContext | undefined; + const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => { + const isStartUrl = context.request.url === url; - if (isStartUrl) await context.addRequests([`${url}/1`]); - else requestContext = context; - }; + if (isStartUrl) await context.addRequests([`${url}/1`]); + else requestContext = context; + }; - const crawler = await makeOneshotCrawler( - { requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 }, - [], - ); + const crawler = await makeOneshotCrawler( + { requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 }, + [], + ); - await crawler.run([{ url, crawlDepth: 2 }]); + await crawler.run([{ url, crawlDepth: 2 }]); - assert(requestContext); - expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 }); - }, - ); + assert(requestContext); + expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 }); + }); describe('should enqueue links correctly', () => { test.each([ @@ -315,49 +315,49 @@ describe('AdaptivePlaywrightCrawler', () => { }); }); - test.each([['static'], ['clientOnly']] as const)( - 'should respect the strategy option for enqueueLinks (%s)', - async (renderingType) => { - const renderingTypePredictor = makeRiggedRenderingTypePredictor({ - detectionProbabilityRecommendation: 0, - renderingType, - }); - const url = new URL(`http://${HOSTNAME}:${port}/external-links`); - const enqueuedUrls = new Set(); - const visitedUrls = new Set(); + test.each([ + ['static'], + ['clientOnly'], + ] as const)('should respect the strategy option for enqueueLinks (%s)', async (renderingType) => { + const renderingTypePredictor = makeRiggedRenderingTypePredictor({ + detectionProbabilityRecommendation: 0, + renderingType, + }); + const url = new URL(`http://${HOSTNAME}:${port}/external-links`); + const enqueuedUrls = new Set(); + const visitedUrls = new Set(); - const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn( - async ({ enqueueLinks, request }) => { - visitedUrls.add(request.loadedUrl); + const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn( + async ({ enqueueLinks, request }) => { + visitedUrls.add(request.loadedUrl); - if (!request.label) { - const result = await enqueueLinks({ - label: 'enqueued-url', - strategy: 'same-hostname', - }); + if (!request.label) { + const result = await enqueueLinks({ + label: 'enqueued-url', + strategy: 'same-hostname', + }); - for (const processedRequest of result.processedRequests) { - enqueuedUrls.add(processedRequest.uniqueKey); - } + for (const processedRequest of result.processedRequests) { + enqueuedUrls.add(processedRequest.uniqueKey); } - }, - ); + } + }, + ); - const crawler = await makeOneshotCrawler( - { - requestHandler, - renderingTypePredictor, - maxRequestsPerCrawl: 10, - }, - [url.toString()], - ); + const crawler = await makeOneshotCrawler( + { + requestHandler, + renderingTypePredictor, + maxRequestsPerCrawl: 10, + }, + [url.toString()], + ); - await crawler.run(); + await crawler.run(); - expect(new Set(visitedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-links`])); - expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-redirect`])); - }, - ); + expect(new Set(visitedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-links`])); + expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-redirect`])); + }); test('should persist crawler state', async () => { const renderingTypePredictor = makeRiggedRenderingTypePredictor({ diff --git a/test/core/crawlers/basic_browser_crawler.ts b/test/core/crawlers/basic_browser_crawler.ts index 620752da39a8..20aeaff759da 100644 --- a/test/core/crawlers/basic_browser_crawler.ts +++ b/test/core/crawlers/basic_browser_crawler.ts @@ -1,15 +1,30 @@ -import type { PuppeteerPlugin } from '@crawlee/browser-pool'; -import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext, PuppeteerGoToOptions } from '@crawlee/puppeteer'; +import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool'; +import type { + BrowserCrawlerOptions, + BrowserCrawlingContext, + PuppeteerCrawlingContext, + PuppeteerGoToOptions, +} from '@crawlee/puppeteer'; import { BrowserCrawler } from '@crawlee/puppeteer'; -import type { HTTPResponse, LaunchOptions } from 'puppeteer'; +import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer'; + +export type TestCrawlingContext = BrowserCrawlingContext; export class BrowserCrawlerTest extends BrowserCrawler< + Page, + HTTPResponse, + PuppeteerController, { browserPlugins: [PuppeteerPlugin] }, LaunchOptions, - PuppeteerCrawlingContext + TestCrawlingContext > { - constructor(options: Partial = {}) { - super(options as any); + constructor( + options: Partial> = {}, + ) { + super({ + ...options, + contextPipelineBuilder: () => this.buildContextPipeline(), + }); } protected async _navigationHandler( diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 5ab04a28627f..0b33b15a7a8c 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -27,14 +27,14 @@ import { RequestState } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; import { RobotsTxtFile, sleep } from '@crawlee/utils'; import express from 'express'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import type { SetRequired } from 'type-fest'; import type { Mock } from 'vitest'; -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, test, vitest } from 'vitest'; import log from '@apify/log'; -import { startExpressAppPromise } from '../../shared/_helper'; +import { startExpressAppPromise } from '../../shared/_helper.js'; describe('BasicCrawler', () => { let logLevel: number; @@ -392,9 +392,9 @@ describe('BasicCrawler', () => { const processed: { url: string }[] = []; const requestList = await RequestList.open(null, sources); - const requestHandler: RequestHandler = async ({ request, crawler }) => { + const requestHandler: RequestHandler = async ({ request, useState }) => { await sleep(10); - const state = await crawler.useState({ processed }); + const state = await useState({ processed }); state.processed.push({ url: request.url }); }; @@ -414,61 +414,116 @@ describe('BasicCrawler', () => { expect(await requestList.isEmpty()).toBe(true); }); - test.each([EventType.MIGRATING, EventType.ABORTING])( - 'should pause on %s event and persist RequestList state', - async (event) => { - const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` })); - - let persistResolve!: (value?: unknown) => void; - const persistPromise = new Promise((res) => { - persistResolve = res; + test('print a warning on sharing state between two crawlers', async () => { + function createCrawler() { + return new BasicCrawler({ + requestHandler: async ({ request, useState }) => { + const state = await useState<{ urls: string[] }>({ urls: [] }); + state.urls.push(request.url); + }, }); + } - // Mock the calls to persist sources. - const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue'); - const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue'); - getValueSpy.mockResolvedValue(null); + const loggerSpy = vitest.spyOn(log, 'warning'); - const processed: { url: string }[] = []; - const requestList = await RequestList.open('reqList', sources); - const requestHandler: RequestHandler = async ({ request }) => { - if (request.url.endsWith('200')) events.emit(event); - processed.push({ url: request.url }); - }; + const [crawler1, crawler2] = [createCrawler(), createCrawler()]; - const basicCrawler = new BasicCrawler({ - requestList, - minConcurrency: 25, - maxConcurrency: 25, - requestHandler, - }); + await crawler1.run([`http://${HOSTNAME}:${port}/`]); + await crawler2.run([`http://${HOSTNAME}:${port}/?page=2`]); + + // Both crawlers should share the same state (backward compatibility) + const state1 = await crawler1.useState<{ urls: string[] }>(); + const state2 = await crawler2.useState<{ urls: string[] }>(); - let finished = false; - // Mock the call to persist state. - setValueSpy.mockImplementationOnce(persistResolve as any); - // The crawler will pause after 200 requests - const runPromise = basicCrawler.run(); - void runPromise.then(() => { - finished = true; + expect(state1).toBe(state2); + expect(state1.urls).toHaveLength(2); + expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/`); + expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/?page=2`); + expect(loggerSpy).toBeCalledWith(expect.stringContaining('Multiple crawler instances are calling useState()')); + }); + + test('crawlers with explicit id have isolated state', async () => { + function createCrawler(id: string) { + return new BasicCrawler({ + id, + requestHandler: async ({ request, useState }) => { + const state = await useState<{ urls: string[] }>({ urls: [] }); + state.urls.push(request.url); + }, }); + } + + const [crawler1, crawler2] = [createCrawler('crawler-1'), createCrawler('crawler-2')]; + + await crawler1.run([`http://${HOSTNAME}:${port}/`]); + await crawler2.run([`http://${HOSTNAME}:${port}/?page=2`]); - // need to monkeypatch the stats class, otherwise it will never finish - basicCrawler.stats.persistState = async () => Promise.resolve(); - await persistPromise; + // Each crawler should have its own isolated state + const state1 = await crawler1.useState<{ urls: string[] }>(); + const state2 = await crawler2.useState<{ urls: string[] }>(); - expect(finished).toBe(false); - expect(await requestList.isFinished()).toBe(false); - expect(await requestList.isEmpty()).toBe(false); - expect(processed.length).toBe(200); + expect(state1).not.toBe(state2); + expect(state1.urls).toHaveLength(1); + expect(state1.urls).toContain(`http://${HOSTNAME}:${port}/`); + expect(state2.urls).toHaveLength(1); + expect(state2.urls).toContain(`http://${HOSTNAME}:${port}/?page=2`); + }); + + test.each([ + EventType.MIGRATING, + EventType.ABORTING, + ])('should pause on %s event and persist RequestList state', async (event) => { + const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` })); - expect(getValueSpy).toBeCalled(); - expect(setValueSpy).toBeCalled(); + let persistResolve!: (value?: unknown) => void; + const persistPromise = new Promise((res) => { + persistResolve = res; + }); - // clean up - // @ts-expect-error Accessing private method - await basicCrawler.autoscaledPool!._destroy(); - }, - ); + // Mock the calls to persist sources. + const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue'); + const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue'); + getValueSpy.mockResolvedValue(null); + + const processed: { url: string }[] = []; + const requestList = await RequestList.open('reqList', sources); + const requestHandler: RequestHandler = async ({ request }) => { + if (request.url.endsWith('200')) events.emit(event); + processed.push({ url: request.url }); + }; + + const basicCrawler = new BasicCrawler({ + requestList, + minConcurrency: 25, + maxConcurrency: 25, + requestHandler, + }); + + let finished = false; + // Mock the call to persist state. + setValueSpy.mockImplementationOnce(persistResolve as any); + // The crawler will pause after 200 requests + const runPromise = basicCrawler.run(); + void runPromise.then(() => { + finished = true; + }); + + // need to monkeypatch the stats class, otherwise it will never finish + basicCrawler.stats.persistState = async () => Promise.resolve(); + await persistPromise; + + expect(finished).toBe(false); + expect(await requestList.isFinished()).toBe(false); + expect(await requestList.isEmpty()).toBe(false); + expect(processed.length).toBe(200); + + expect(getValueSpy).toBeCalled(); + expect(setValueSpy).toBeCalled(); + + // clean up + // @ts-expect-error Accessing private method + await basicCrawler.autoscaledPool!._destroy(); + }); test('should retry failed requests', async () => { const sources = [ @@ -1160,14 +1215,14 @@ describe('BasicCrawler', () => { vitest.restoreAllMocks(); }); - test('should timeout after handleRequestTimeoutSecs', async () => { + test('should timeout after requestHandlerTimeoutSecs', async () => { const url = 'https://example.com'; const requestList = await RequestList.open({ sources: [{ url }] }); const results: Request[] = []; const crawler = new BasicCrawler({ requestList, - handleRequestTimeoutSecs: 0.01, + requestHandlerTimeoutSecs: 0.01, maxRequestRetries: 1, requestHandler: async () => sleep(1000), failedRequestHandler: async ({ request }) => { @@ -1181,7 +1236,7 @@ describe('BasicCrawler', () => { results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out')); }); - test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => { + test('limits requestHandlerTimeoutSecs and derived vars to a valid value', async () => { const url = 'https://example.com'; const requestList = await RequestList.open({ sources: [{ url }] }); @@ -1223,9 +1278,9 @@ describe('BasicCrawler', () => { for (const args of warningSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true); - expect(/requestHandler timed out after/.test(args[0])).toBe(true); - expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false); + expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/); + expect(args[0]).toMatch(/requestHandler timed out after/); + expect(args[0]).not.toMatch(/at Timeout\._onTimeout/); expect(args[1]).toBeDefined(); } @@ -1233,9 +1288,9 @@ describe('BasicCrawler', () => { for (const args of errorSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true); - expect(/requestHandler timed out after/.test(args[0])).toBe(true); - expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false); + expect(args[0]).toMatch(/Request failed and reached maximum retries/); + expect(args[0]).toMatch(/requestHandler timed out after/); + expect(args[0]).not.toMatch(/at Timeout\._onTimeout/); expect(args[1]).toBeDefined(); } }); @@ -1261,8 +1316,8 @@ describe('BasicCrawler', () => { for (const args of warningSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true); - expect(/Other non-timeout error/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/); + expect(args[0]).toMatch(/Other non-timeout error/); expect(args[0].split('\n').length).toBeLessThanOrEqual(2); expect(args[1]).toBeDefined(); } @@ -1271,9 +1326,9 @@ describe('BasicCrawler', () => { for (const args of errorSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true); - expect(/Other non-timeout error/.test(args[0])).toBe(true); - expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Request failed and reached maximum retries/); + expect(args[0]).toMatch(/Other non-timeout error/); + expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/); expect(args[1]).toBeDefined(); } }); @@ -1300,9 +1355,9 @@ describe('BasicCrawler', () => { for (const args of warningSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true); - expect(/requestHandler timed out after/.test(args[0])).toBe(true); - expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/); + expect(args[0]).toMatch(/requestHandler timed out after/); + expect(args[0]).toMatch(/at Timeout\._onTimeout/); expect(args[1]).toBeDefined(); } @@ -1310,9 +1365,9 @@ describe('BasicCrawler', () => { for (const args of errorSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true); - expect(/requestHandler timed out after/.test(args[0])).toBe(true); - expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Request failed and reached maximum retries/); + expect(args[0]).toMatch(/requestHandler timed out after/); + expect(args[0]).toMatch(/at Timeout\._onTimeout/); expect(args[1]).toBeDefined(); } @@ -1343,9 +1398,9 @@ describe('BasicCrawler', () => { for (const args of warningSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true); - expect(/Other non-timeout error/.test(args[0])).toBe(true); - expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Reclaiming failed request back to the list or queue/); + expect(args[0]).toMatch(/Other non-timeout error/); + expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/); expect(args[1]).toBeDefined(); } @@ -1353,9 +1408,9 @@ describe('BasicCrawler', () => { for (const args of errorSpy.mock.calls) { expect(args.length).toBe(2); expect(typeof args[0]).toBe('string'); - expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true); - expect(/Other non-timeout error/.test(args[0])).toBe(true); - expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true); + expect(args[0]).toMatch(/Request failed and reached maximum retries/); + expect(args[0]).toMatch(/Other non-timeout error/); + expect(args[0]).toMatch(/at _?BasicCrawler\.requestHandler/); expect(args[1]).toBeDefined(); } @@ -1371,7 +1426,7 @@ describe('BasicCrawler', () => { const crawler = new BasicCrawler({ requestList, - handleRequestTimeoutSecs: 0.01, + requestHandlerTimeoutSecs: 0.01, maxRequestRetries: 1, useSessionPool: true, sessionPoolOptions: { @@ -1398,7 +1453,7 @@ describe('BasicCrawler', () => { const crawler = new BasicCrawler({ requestList, - handleRequestTimeoutSecs: 0.01, + requestHandlerTimeoutSecs: 0.01, maxRequestRetries: 1, useSessionPool: true, sessionPoolOptions: { @@ -1421,7 +1476,7 @@ describe('BasicCrawler', () => { const crawler = new BasicCrawler({ requestList, - handleRequestTimeoutSecs: 0.01, + requestHandlerTimeoutSecs: 0.01, maxRequestRetries: 1, useSessionPool: true, sessionPoolOptions: { @@ -1444,50 +1499,20 @@ describe('BasicCrawler', () => { }); }); - describe('CrawlingContext', () => { - test('should be kept and later deleted', async () => { - const urls = [ - 'https://example.com/0', - 'https://example.com/1', - 'https://example.com/2', - 'https://example.com/3', - ]; - const requestList = await RequestList.open(null, urls); - let counter = 0; - let finish: (value?: unknown) => void; - const allFinishedPromise = new Promise((resolve) => { - finish = resolve; - }); - const mainContexts: CrawlingContext[] = []; - const otherContexts: CrawlingContext[][] = []; - const crawler = new BasicCrawler({ - requestList, - minConcurrency: 4, - async requestHandler(crawlingContext) { - // @ts-expect-error Accessing private prop - mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id); - // @ts-expect-error Accessing private prop - otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v); - counter++; - if (counter === 4) finish(); - await allFinishedPromise; - }, - }); - await crawler.run(); + test('extendContext', async () => { + const url = 'https://example.com'; + const requestHandlerImplementation = vi.fn(); - expect(counter).toBe(4); - expect(mainContexts).toHaveLength(4); - expect(otherContexts).toHaveLength(4); - // @ts-expect-error Accessing private prop - expect(crawler.crawlingContexts.size).toBe(0); - mainContexts.forEach((ctx, idx) => { - expect(typeof ctx.id).toBe('string'); - expect(otherContexts[idx]).toContain(ctx); - }); - otherContexts.forEach((list, idx) => { - expect(list).toHaveLength(idx + 1); - }); + const crawler = new BasicCrawler({ + extendContext: () => ({ hello: 'world' }), + requestHandler: async ({ hello }) => { + requestHandlerImplementation({ hello }); + }, }); + + await crawler.run([url]); + expect(requestHandlerImplementation).toHaveBeenCalledOnce(); + expect(requestHandlerImplementation.mock.calls[0][0]).toMatchObject({ hello: 'world' }); }); describe('sendRequest', () => { @@ -1526,8 +1551,8 @@ describe('BasicCrawler', () => { const response = await sendRequest(); responses.push({ - statusCode: response.statusCode, - body: response.body, + statusCode: response.status, + body: await response.text(), }); }, }); @@ -1554,8 +1579,8 @@ describe('BasicCrawler', () => { const response = await sendRequest(); responses.push({ - statusCode: response.statusCode, - body: response.body, + statusCode: response.status, + body: await response.text(), }); }, }); @@ -1819,7 +1844,7 @@ describe('BasicCrawler', () => { const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }]; const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }]; - const tmpDir = `${__dirname}/tmp/foo/bar`; + const tmpDir = `${import.meta.dirname}/tmp/foo/bar`; beforeAll(async () => { await rm(tmpDir, { recursive: true, force: true }); diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts index 6075c1c2c0bd..f808dc253aa2 100644 --- a/test/core/crawlers/browser_crawler.test.ts +++ b/test/core/crawlers/browser_crawler.test.ts @@ -1,27 +1,20 @@ import type { Server } from 'node:http'; -import { BROWSER_POOL_EVENTS, BrowserPool, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool'; +import { BROWSER_POOL_EVENTS, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool'; import { BLOCKED_STATUS_CODES } from '@crawlee/core'; -import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, PuppeteerRequestHandler } from '@crawlee/puppeteer'; -import { - AutoscaledPool, - EnqueueStrategy, - ProxyConfiguration, - Request, - RequestList, - RequestState, - Session, -} from '@crawlee/puppeteer'; +import type { PuppeteerGoToOptions } from '@crawlee/puppeteer'; +import { EnqueueStrategy, ProxyConfiguration, Request, RequestList, RequestState, Session } from '@crawlee/puppeteer'; import { sleep } from '@crawlee/utils'; import type { HTTPResponse } from 'puppeteer'; import puppeteer from 'puppeteer'; -import { runExampleComServer } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { runExampleComServer } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import { ENV_VARS } from '@apify/consts'; import log from '@apify/log'; -import { BrowserCrawlerTest } from './basic_browser_crawler'; +import type { TestCrawlingContext } from './basic_browser_crawler.js'; +import { BrowserCrawlerTest } from './basic_browser_crawler.js'; describe('BrowserCrawler', () => { let prevEnvHeadless: string; @@ -72,7 +65,7 @@ describe('BrowserCrawler', () => { const processed: Request[] = []; const failed: Request[] = []; const requestList = await RequestList.open(null, sources); - const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => { + const requestHandler = async ({ page, request, response }: TestCrawlingContext) => { await page.waitForSelector('title'); expect(response!.status()).toBe(200); @@ -132,7 +125,7 @@ describe('BrowserCrawler', () => { let sessionGoto!: Session; const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, ): Promise { vitest.spyOn(ctx.session!, 'markBad'); sessionGoto = ctx.session!; @@ -156,17 +149,12 @@ describe('BrowserCrawler', () => { const requestList = await RequestList.open({ sources: [{ url: 'http://example.com/?q=1' }], }); - let isEvaluated = false; - const browserCrawler = new (class extends BrowserCrawlerTest { - protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, - gotoOptions: PuppeteerGoToOptions, - ): Promise { - isEvaluated = ctx.hookFinished as boolean; - return ctx.page.goto(ctx.request.url, gotoOptions); - } - })({ + const hook = vi.fn(async () => { + await sleep(10); + }); + + const browserCrawler = new BrowserCrawlerTest({ browserPoolOptions: { browserPlugins: [puppeteerPlugin], }, @@ -174,24 +162,22 @@ describe('BrowserCrawler', () => { useSessionPool: true, requestHandler: async () => {}, maxRequestRetries: 0, - preNavigationHooks: [ - async (crawlingContext) => { - await sleep(10); - crawlingContext.hookFinished = true; - }, - ], + preNavigationHooks: [hook], }); await browserCrawler.run(); - expect(isEvaluated).toBeTruthy(); + expect(hook).toHaveBeenCalled(); }); test('should evaluate postNavigationHooks', async () => { const requestList = await RequestList.open({ sources: [{ url: `${serverAddress}/?q=1` }], }); - let isEvaluated = false; + + const hook = vi.fn(async () => { + await sleep(10); + }); const browserCrawler = new BrowserCrawlerTest({ browserPoolOptions: { @@ -199,21 +185,14 @@ describe('BrowserCrawler', () => { }, requestList, useSessionPool: true, - requestHandler: async ({ hookFinished }) => { - isEvaluated = hookFinished as boolean; - }, + requestHandler: async () => {}, maxRequestRetries: 0, - postNavigationHooks: [ - async (crawlingContext) => { - await sleep(10); - crawlingContext.hookFinished = true; - }, - ], + postNavigationHooks: [hook], }); await browserCrawler.run(); - expect(isEvaluated).toBeTruthy(); + expect(hook).toHaveBeenCalled(); }); test('errorHandler has open page', async () => { @@ -233,7 +212,7 @@ describe('BrowserCrawler', () => { }, maxRequestRetries: 1, errorHandler: async (ctx, error) => { - result.push(await ctx.page.evaluate(() => window.location.origin)); + result.push(await ctx.page!.evaluate(() => window.location.origin)); }, }); @@ -293,7 +272,7 @@ describe('BrowserCrawler', () => { let optionsGoto: PuppeteerGoToOptions; const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, gotoOptions: PuppeteerGoToOptions, ): Promise { optionsGoto = gotoOptions; @@ -364,7 +343,7 @@ describe('BrowserCrawler', () => { requestList, requestHandler: async () => { setTimeout(() => callSpy('good'), 300); - setTimeout(() => callSpy('bad'), 1500); + setTimeout(() => callSpy('bad'), 2500); await new Promise(() => {}); }, requestHandlerTimeoutSecs: 0.5, @@ -640,9 +619,9 @@ describe('BrowserCrawler', () => { let called = false; const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, ): Promise { - ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => { + browserCrawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => { resolve(); called = true; }); @@ -864,10 +843,9 @@ describe('BrowserCrawler', () => { const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, ): Promise { - const { session } = ctx; - const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); + const proxyInfo = ctx.session?.proxyInfo; if (proxyInfo!.url !== goodProxyUrl) { throw new Error('ERR_PROXY_CONNECTION_FAILED'); @@ -903,10 +881,9 @@ describe('BrowserCrawler', () => { let numberOfRotations = -requestList!.length(); const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, ): Promise { - const { session } = ctx; - const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); + const proxyInfo = ctx.session?.proxyInfo; numberOfRotations++; @@ -941,10 +918,9 @@ describe('BrowserCrawler', () => { const crawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( - ctx: PuppeteerCrawlingContext, + ctx: TestCrawlingContext, ): Promise { - const { session } = ctx; - const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); + const proxyInfo = ctx.session?.proxyInfo; if (proxyInfo!.url.includes('localhost')) { throw new Error(proxyError); @@ -990,39 +966,32 @@ describe('BrowserCrawler', () => { }); test('uses correct crawling context', async () => { - let prepareCrawlingContext: PuppeteerCrawlingContext; + let prepareCrawlingContext: TestCrawlingContext; - const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => { + const gotoFunction = async (crawlingContext: TestCrawlingContext) => { prepareCrawlingContext = crawlingContext; expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); expect(typeof crawlingContext.page).toBe('object'); }; - const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => { + const requestHandler = async (crawlingContext: TestCrawlingContext) => { expect(crawlingContext === prepareCrawlingContext).toEqual(true); expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); expect(typeof crawlingContext.page).toBe('object'); - expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest); expect(Object.hasOwn(crawlingContext, 'response')).toBe(true); throw new Error('some error'); }; - const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => { + const failedRequestHandler = async (crawlingContext: Partial, error: Error) => { expect(crawlingContext).toBe(prepareCrawlingContext); expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); expect(typeof crawlingContext.page).toBe('object'); - expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest); - expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool); expect(Object.hasOwn(crawlingContext, 'response')).toBe(true); - expect(crawlingContext.error).toBeInstanceOf(Error); expect(error).toBeInstanceOf(Error); expect(error.message).toEqual('some error'); }; diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts index b73065804040..30bb5ec03b76 100644 --- a/test/core/crawlers/cheerio_crawler.test.ts +++ b/test/core/crawlers/cheerio_crawler.test.ts @@ -1,20 +1,8 @@ -import type { IncomingHttpHeaders, Server } from 'node:http'; -import { Readable } from 'node:stream'; - -import type { - Cheerio, - CheerioAPI, - CheerioCrawlingContext, - CheerioRequestHandler, - CheerioRoot, - Element, - ProxyInfo, - Source, -} from '@crawlee/cheerio'; +import type { Server } from 'node:http'; + +import type { BasicCrawlingContext, CheerioCrawlingContext, CheerioRequestHandler, Source } from '@crawlee/cheerio'; import { - AutoscaledPool, CheerioCrawler, - CrawlerExtension, createCheerioRouter, EnqueueStrategy, mergeCookies, @@ -23,13 +11,14 @@ import { RequestList, Session, } from '@crawlee/cheerio'; +import { ImpitHttpClient } from '@crawlee/impit-client'; +import type { ProxyInfo } from '@crawlee/types'; import type { Dictionary } from '@crawlee/utils'; import { sleep } from '@crawlee/utils'; -// @ts-expect-error type import of ESM only package import type { OptionsInit } from 'got-scraping'; import iconv from 'iconv-lite'; -import { responseSamples, runExampleComServer } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { responseSamples, runExampleComServer } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import log, { Log } from '@apify/log'; @@ -50,12 +39,12 @@ async function getRequestListForMock(mockData: Dictionary, pathName = 'special/m return requestList; } -async function getRequestListForMirror() { +async function getExampleRequestList(pathname = '/special/mirror') { const sources = [ - { url: `${serverAddress}/special/mirror?a=12` }, - { url: `${serverAddress}/special/mirror?a=23` }, - { url: `${serverAddress}/special/mirror?a=33` }, - { url: `${serverAddress}/special/mirror?a=43` }, + { url: `${serverAddress}${pathname}?a=12` }, + { url: `${serverAddress}${pathname}?a=23` }, + { url: `${serverAddress}${pathname}?a=33` }, + { url: `${serverAddress}${pathname}?a=43` }, ]; const requestList = await RequestList.open(null, sources); return requestList; @@ -66,6 +55,10 @@ beforeAll(async () => { serverAddress += port; }); +afterEach(() => { + vi.useRealTimers(); +}); + afterAll(() => { server.close(); }); @@ -92,7 +85,7 @@ describe('CheerioCrawler', () => { }); test('should work', async () => { - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const processed: Request[] = []; const failed: Request[] = []; const requestHandler: CheerioRequestHandler = ({ $, body, request }) => { @@ -125,7 +118,7 @@ describe('CheerioCrawler', () => { }); test('should work with implicit router', async () => { - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const processed: Request[] = []; const failed: Request[] = []; @@ -158,7 +151,7 @@ describe('CheerioCrawler', () => { }); test('should work with explicit router', async () => { - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const processed: Request[] = []; const failed: Request[] = []; @@ -194,7 +187,7 @@ describe('CheerioCrawler', () => { }); test('should throw when no requestHandler nor default route provided', async () => { - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const cheerioCrawler = new CheerioCrawler({ requestList, @@ -272,7 +265,8 @@ describe('CheerioCrawler', () => { maxRequestRetries: 0, maxConcurrency: 1, requestHandler: ({ $, body, request }) => { - tmp.push(body, $.html(), request.loadedUrl); + // test that `request.loadedUrl` is no longer optional by calling `toLowerCase` on it directly (no optional chaining) + tmp.push(body, $.html(), request.loadedUrl.toLowerCase()); }, }); @@ -281,7 +275,6 @@ describe('CheerioCrawler', () => { expect(tmp).toHaveLength(3); expect(tmp[0]).toBe(responseSamples.html); expect(tmp[1]).toBe(tmp[0]); - // test that `request.loadedUrl` is no longer optional expect(tmp[2].length).toBe(sources[0].length); }); @@ -341,10 +334,10 @@ describe('CheerioCrawler', () => { test('after requestHandlerTimeoutSecs', async () => { const failed: Request[] = []; - const requestList = await getRequestListForMirror(); - const requestHandler = async () => { + const requestList = await getExampleRequestList(); + const requestHandler = vi.fn(async () => { await sleep(2000); - }; + }); const cheerioCrawler = new CheerioCrawler({ requestList, @@ -358,18 +351,20 @@ describe('CheerioCrawler', () => { }, }); - // Override low value to prevent seeing timeouts from BasicCrawler - // @ts-expect-error Overriding private property - cheerioCrawler.handleRequestTimeoutMillis = 10000; - await cheerioCrawler.run(); + expect(requestHandler).toHaveBeenCalledTimes(8); expect(failed).toHaveLength(4); failed.forEach((request) => { - expect(request.errorMessages).toHaveLength(2); - expect(request.errorMessages[0]).toMatch('requestHandler timed out'); - expect(request.errorMessages[1]).toMatch('requestHandler timed out'); + expect(request).toEqual( + expect.objectContaining({ + errorMessages: [ + expect.stringContaining('requestHandler timed out'), + expect.stringContaining('requestHandler timed out'), + ], + }), + ); }); }); }); @@ -405,19 +400,19 @@ describe('CheerioCrawler', () => { describe('should ensure text/html Content-Type', () => { test('by setting a correct Accept header', async () => { - const headers: IncomingHttpHeaders[] = []; - const requestList = await getRequestListForMirror(); + const headersPerRequests: Headers[] = []; + const requestList = await getExampleRequestList('/special/headers'); const crawler = new CheerioCrawler({ requestList, - requestHandler: ({ response }) => { - headers.push(response.request.options.headers); + requestHandler: async ({ json }) => { + headersPerRequests.push(new Headers(json.headers)); }, }); await crawler.run(); - expect(headers).toHaveLength(4); - headers.forEach((h) => { - const acceptHeader = h.accept || h.Accept; + expect(headersPerRequests).toHaveLength(4); + headersPerRequests.forEach((headerset) => { + const acceptHeader = headerset.get('accept'); expect(acceptHeader!.includes('text/html')).toBe(true); expect(acceptHeader!.includes('application/xhtml+xml')).toBe(true); }); @@ -542,7 +537,7 @@ describe('CheerioCrawler', () => { }); test('should throw an error on http error status codes set by user', async () => { - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const failed: Request[] = []; const cheerioCrawler = new CheerioCrawler({ @@ -608,7 +603,7 @@ describe('CheerioCrawler', () => { const url = `${serverAddress}/special/json-type`; await runCrawler(url); expect(handlePageInvocationParams.json).toBeInstanceOf(Object); - expect(handlePageInvocationParams.body).toEqual(Buffer.from(JSON.stringify(responseSamples.json))); + expect(handlePageInvocationParams.body).toEqual(JSON.stringify(responseSamples.json)); expect(handlePageInvocationParams.contentType.type).toBe('application/json'); expect(handleFailedInvoked).toBe(false); }); @@ -623,8 +618,8 @@ describe('CheerioCrawler', () => { test('when response is image/png', async () => { const url = `${serverAddress}/special/image-type`; await runCrawler(url); - expect(handlePageInvocationParams.body).toBeInstanceOf(Buffer); - expect(handlePageInvocationParams.body).toEqual(responseSamples.image); + expect(typeof handlePageInvocationParams.body).toBe('string'); + expect(handlePageInvocationParams.body).toEqual(responseSamples.image.toString()); expect(handlePageInvocationParams.contentType.type).toBe('image/png'); }); }); @@ -647,15 +642,10 @@ describe('CheerioCrawler', () => { suggestResponseEncoding, }); - const stream = Readable.from([buf]); - // @ts-expect-error Using private method - const { response, encoding } = crawler._encodeResponse({}, stream); + const { response, encoding } = crawler._encodeResponse({}, new Response(new Uint8Array(buf))); expect(encoding).toBe('utf8'); - for await (const chunk of response) { - const string = chunk.toString('utf8'); - expect(string).toBe(html); - } + expect(await response.text()).toBe(html); }); test('always when forced', async () => { @@ -673,15 +663,10 @@ describe('CheerioCrawler', () => { forceResponseEncoding, }); - const stream = Readable.from([buf]); - // @ts-expect-error Using private method - const { response, encoding } = crawler._encodeResponse({}, stream, 'ascii'); + const { response, encoding } = crawler._encodeResponse({}, new Response(new Uint8Array(buf)), 'ascii'); expect(encoding).toBe('utf8'); - for await (const chunk of response) { - const string = chunk.toString('utf8'); - expect(string).toBe(html); - } + expect(await response.text()).toBe(html); }); test('Cheerio decodes html entities', async () => { @@ -697,7 +682,7 @@ describe('CheerioCrawler', () => { context = context as unknown as CheerioCrawlingContext; expect(context?.$.html()).toBe('"<>"<>'); - expect(context?.$.html({ decodeEntities: false })).toBe('"<>"<>'); + expect(context?.$.html({ xml: { decodeEntities: false, xmlMode: false } })).toBe('"<>"<>'); expect(context?.body).toBe('"<>"<>'); }); }); @@ -714,7 +699,7 @@ describe('CheerioCrawler', () => { proxyUrls: [proxyUrl], }); - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const proxies: string[] = []; const crawler = new CheerioCrawler({ @@ -746,7 +731,7 @@ describe('CheerioCrawler', () => { sessions.push(session!); }; - const requestList = await getRequestListForMirror(); + const requestList = await getExampleRequestList(); const crawler = new CheerioCrawler({ requestList, @@ -762,8 +747,7 @@ describe('CheerioCrawler', () => { const session = sessions[i]; expect(typeof proxyInfo.url).toBe('string'); expect(typeof session.id).toBe('string'); - expect(proxyInfo.sessionId).toBe(session.id); - expect(proxyInfo).toEqual(await proxyConfiguration.newProxyInfo(session.id)); + expect(session.proxyInfo).toBe(proxyInfo); } }); @@ -806,20 +790,22 @@ describe('CheerioCrawler', () => { */ let numberOfRotations = -1; const failedRequestHandler = vitest.fn(); + const impit = new ImpitHttpClient(); const crawler = new CheerioCrawler({ proxyConfiguration, maxSessionRotations: 5, requestHandler: async () => {}, failedRequestHandler, - }); - - vitest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => { - if (proxyUrl.includes('localhost')) { - numberOfRotations++; - throw new Error('Proxy responded with 400 - Bad request'); - } - - return null; + httpClient: { + sendRequest: async (request, opts) => { + const { session } = opts ?? {}; + if (session?.proxyInfo?.url.includes('localhost')) { + numberOfRotations++; + throw new Error('Proxy responded with 400 - Bad request'); + } + return await impit.sendRequest(request); + }, + }, }); await crawler.run([serverAddress]); @@ -833,26 +819,28 @@ describe('CheerioCrawler', () => { const proxyError = 'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.'; + const impit = new ImpitHttpClient(); + const crawler = new CheerioCrawler({ proxyConfiguration, maxSessionRotations: 1, requestHandler: async () => {}, - }); - - vitest.spyOn(crawler, '_requestAsBrowser' as any).mockImplementation(async ({ proxyUrl }: any) => { - if (proxyUrl.includes('localhost')) { - throw new Error(proxyError); - } - - return null; + httpClient: { + sendRequest: async (request, opts) => { + const { session } = opts ?? {}; + if (session?.proxyInfo?.url.includes('localhost')) { + throw new Error(proxyError); + } + return impit.sendRequest(request); + }, + }, }); const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {}); await crawler.run([serverAddress]); - expect(spy).toBeCalled(); - expect(spy.mock.calls[0][0]).toEqual(expect.stringContaining(proxyError)); + expect(spy).toHaveBeenCalledWith(expect.stringContaining(proxyError), expect.any(Object)); }); }); @@ -1021,9 +1009,8 @@ describe('CheerioCrawler', () => { }); }); - test('should merge cookies set in pre-nav hook with the session ones', async () => { + test('should merge request and session cookies', async () => { const responses: unknown[] = []; - const gotOptions: OptionsInit[] = []; const crawler = new CheerioCrawler({ requestList: await RequestList.open(null, [ { @@ -1039,11 +1026,6 @@ describe('CheerioCrawler', () => { requestHandler: ({ json }) => { responses.push(json); }, - preNavigationHooks: [ - (_context, options) => { - gotOptions.push(options); - }, - ], }); const sessSpy = vitest.spyOn(Session.prototype, 'getCookieString'); @@ -1055,12 +1037,6 @@ describe('CheerioCrawler', () => { cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123', }, }); - expect(gotOptions).toHaveLength(1); - expect(gotOptions[0]).toMatchObject({ - headers: { - Cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123', // header name normalized to `Cookie` - }, - }); }); test('should work with cookies adjusted on `context.request` in pre-nav hook', async () => { @@ -1099,6 +1075,8 @@ describe('CheerioCrawler', () => { test('should work with `context.request.headers` being undefined', async () => { const requests: Request[] = []; const responses: unknown[] = []; + const errorHandler = vi.fn(async () => {}); + const crawler = new CheerioCrawler({ requestList: await RequestList.open(null, [ { @@ -1110,6 +1088,7 @@ describe('CheerioCrawler', () => { responses.push(json); requests.push(request); }, + errorHandler, preNavigationHooks: [ ({ request }) => { request.headers!.Cookie = 'foo=override; coo=kie'; @@ -1118,6 +1097,9 @@ describe('CheerioCrawler', () => { }); await crawler.run(); + + expect(errorHandler).not.toHaveBeenCalled(); + expect(requests).toHaveLength(1); expect(requests[0].retryCount).toBe(0); expect(responses).toHaveLength(1); @@ -1129,14 +1111,14 @@ describe('CheerioCrawler', () => { }); test('mergeCookies()', async () => { - const deprecatedSpy = vitest.spyOn(Log.prototype, 'deprecated'); + const warningSpy = vitest.spyOn(Log.prototype, 'warningOnce'); const cookie1 = mergeCookies('https://example.com', [ 'foo=bar1; other=cookie1 ; coo=kie', 'foo=bar2; baz=123', 'other=cookie2;foo=bar3', ]); expect(cookie1).toBe('foo=bar3; other=cookie2; coo=kie; baz=123'); - expect(deprecatedSpy).not.toBeCalled(); + expect(warningSpy).not.toBeCalled(); const cookie2 = mergeCookies('https://example.com', [ 'Foo=bar1; other=cookie1 ; coo=kie', @@ -1144,14 +1126,12 @@ describe('CheerioCrawler', () => { 'Other=cookie2;foo=bar3', ]); expect(cookie2).toBe('Foo=bar1; other=cookie1; coo=kie; foo=bar3; baz=123; Other=cookie2'); - expect(deprecatedSpy).toBeCalledTimes(3); - expect(deprecatedSpy).toBeCalledWith( - `Found cookies with similar name during cookie merging: 'foo' and 'Foo'`, - ); - expect(deprecatedSpy).toBeCalledWith( + expect(warningSpy).toBeCalledTimes(3); + expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'foo' and 'Foo'`); + expect(warningSpy).toBeCalledWith( `Found cookies with similar name during cookie merging: 'Other' and 'other'`, ); - deprecatedSpy.mockClear(); + warningSpy.mockClear(); const cookie3 = mergeCookies('https://example.com', [ 'foo=bar1; Other=cookie1 ; Coo=kie', @@ -1159,50 +1139,48 @@ describe('CheerioCrawler', () => { 'Other=cookie2;Foo=bar3;coo=kee', ]); expect(cookie3).toBe('foo=bar2; Other=cookie2; Coo=kie; baz=123; Foo=bar3; coo=kee'); - expect(deprecatedSpy).toBeCalledTimes(2); - expect(deprecatedSpy).toBeCalledWith( - `Found cookies with similar name during cookie merging: 'Foo' and 'foo'`, - ); - expect(deprecatedSpy).toBeCalledWith( - `Found cookies with similar name during cookie merging: 'coo' and 'Coo'`, - ); + expect(warningSpy).toBeCalledTimes(2); + expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Foo' and 'foo'`); + expect(warningSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'coo' and 'Coo'`); }); - test('should use sessionId in proxyUrl when the session pool is enabled', async () => { - const sourcesNew = [{ url: 'http://example.com/?q=1' }]; - const requestListNew = await RequestList.open({ sources: sourcesNew }); - let usedSession: Session; + test('sendRequest and main request should share the same session cookie jar', async () => { + const responses: { cookies: string }[] = []; - const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:8080'] }); - const newUrlSpy = vitest.spyOn(proxyConfiguration, 'newUrl'); - const cheerioCrawler = new CheerioCrawler({ - requestList: requestListNew, - maxRequestRetries: 0, - maxSessionRotations: 0, - requestHandler: () => {}, - failedRequestHandler: () => {}, + const crawler = new CheerioCrawler({ + requestList: await RequestList.open(null, [{ url: `${serverAddress}/special/get-cookies` }]), useSessionPool: true, - proxyConfiguration, - }); + sessionPoolOptions: { + // Even with multiple available sessions, the preNavigationHook should use the same one as the main request + maxPoolSize: 10, + }, + preNavigationHooks: [ + async ({ sendRequest }) => { + await sendRequest({ + url: `${serverAddress}/special/set-cookie?name=sharedCookie&value=sharedValue`, + }); - // @ts-expect-error Accessing private method - const oldHandleRequestF = cheerioCrawler._runRequestHandler; - // @ts-expect-error Overriding private method - cheerioCrawler._runRequestHandler = async (opts) => { - usedSession = opts.session!; - return oldHandleRequestF.call(cheerioCrawler, opts); - }; + const response = await sendRequest({ url: `${serverAddress}/special/get-cookies` }); + const json = await response.json(); - try { - await cheerioCrawler.run(); - } catch (e) { - // localhost proxy causes proxy errors, session rotations and finally throws, but we don't care - } + expect(json.cookies).toContain('sharedCookie=sharedValue'); + }, + ], + requestHandler: async ({ json, sendRequest }) => { + responses.push(json as { cookies: string }); - expect(newUrlSpy).toBeCalledWith( - usedSession!.id, - expect.objectContaining({ request: expect.any(Request) }), - ); + const sendRequestJson = await sendRequest({ url: `${serverAddress}/special/get-cookies` }).then( + async (response) => response.json(), + ); + responses.push(sendRequestJson as { cookies: string }); + }, + }); + + await crawler.run(); + + expect(responses).toHaveLength(2); + expect(responses[0].cookies).toContain('sharedCookie=sharedValue'); + expect(responses[1].cookies).toContain('sharedCookie=sharedValue'); }); }); @@ -1220,19 +1198,17 @@ describe('CheerioCrawler', () => { }); test('uses correct crawling context', async () => { - let prepareCrawlingContext: CheerioCrawlingContext; + let prepareCrawlingContext: unknown; - const prepareRequestFunction = (crawlingContext: CheerioCrawlingContext) => { + const preNavigationHook = (crawlingContext: BasicCrawlingContext) => { prepareCrawlingContext = crawlingContext; expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); }; const requestHandler = (crawlingContext: CheerioCrawlingContext) => { expect(crawlingContext === prepareCrawlingContext).toEqual(true); expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); expect(typeof crawlingContext.$).toBe('function'); expect(typeof crawlingContext.response).toBe('object'); @@ -1241,16 +1217,14 @@ describe('CheerioCrawler', () => { throw new Error('some error'); }; - const failedRequestHandler = (crawlingContext: CheerioCrawlingContext, error: Error) => { + const failedRequestHandler = (crawlingContext: Partial, error: Error) => { expect(crawlingContext === prepareCrawlingContext).toEqual(true); expect(crawlingContext.request).toBeInstanceOf(Request); - expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool); expect(crawlingContext.session).toBeInstanceOf(Session); expect(typeof crawlingContext.$).toBe('function'); expect(typeof crawlingContext.response).toBe('object'); expect(typeof crawlingContext.contentType).toBe('object'); - expect(crawlingContext.error).toBeInstanceOf(Error); expect(error).toBeInstanceOf(Error); expect(error.message).toEqual('some error'); }; @@ -1260,99 +1234,12 @@ describe('CheerioCrawler', () => { maxRequestRetries: 0, maxConcurrency: 1, useSessionPool: true, - preNavigationHooks: [prepareRequestFunction], + preNavigationHooks: [preNavigationHook], requestHandler, failedRequestHandler, }); await cheerioCrawler.run(); }); - - test('should have correct types in crawling context', async () => { - const requestHandler = (crawlingContext: CheerioCrawlingContext) => { - // Checking that types are correct - const _cheerioRootType: CheerioRoot = crawlingContext.$; - const _apiType: CheerioAPI = crawlingContext.$; - const _cheerioElementType: Cheerio = crawlingContext.$('div'); - }; - - const cheerioCrawler = new CheerioCrawler({ - requestList, - maxRequestRetries: 0, - maxConcurrency: 1, - requestHandler, - }); - await cheerioCrawler.run(); - }); - }); - - describe('use', () => { - const sources = ['http://example.com/']; - let requestList: RequestList; - - class DummyExtension extends CrawlerExtension { - constructor(readonly options: Dictionary) { - super(); - } - - override getCrawlerOptions() { - return this.options; - } - } - - beforeEach(async () => { - requestList = await RequestList.open(null, sources.slice()); - }); - - test('should throw if "CrawlerExtension" class is not used', () => { - const cheerioCrawler = new CheerioCrawler({ - requestList, - maxRequestRetries: 0, - requestHandler: () => {}, - failedRequestHandler: () => {}, - }); - expect( - // @ts-expect-error Validating JS side checks - () => cheerioCrawler.use({}), - ).toThrow('Expected object `{}` to be of type `CrawlerExtension`'); - }); - - test('Should throw if "CrawlerExtension" is trying to override non existing property', () => { - const extension = new DummyExtension({ - doesNotExist: true, - }); - const cheerioCrawler = new CheerioCrawler({ - requestList, - maxRequestRetries: 0, - requestHandler: () => {}, - failedRequestHandler: () => {}, - }); - expect(() => cheerioCrawler.use(extension)).toThrow( - 'DummyExtension tries to set property "doesNotExist" that is not configurable on CheerioCrawler instance.', - ); - }); - - test('should override crawler properties', () => { - const extension = new DummyExtension({ - useSessionPool: true, - requestHandler: undefined, - }); - const cheerioCrawler = new CheerioCrawler({ - requestList, - useSessionPool: false, - maxRequestRetries: 0, - requestHandler: () => {}, - failedRequestHandler: () => {}, - }); - // @ts-expect-error Accessing private prop - expect(cheerioCrawler.useSessionPool).toEqual(false); - cheerioCrawler.use(extension); - // @ts-expect-error Accessing private prop - expect(cheerioCrawler.useSessionPool).toEqual(true); - // @ts-expect-error Accessing private prop - expect(cheerioCrawler.requestHandler).toBeUndefined(); - // @ts-expect-error Accessing private prop - expect(cheerioCrawler.requestHandler).toBeUndefined(); - }); }); test('should work with delete requests', async () => { diff --git a/test/core/crawlers/context_pipeline.test.ts b/test/core/crawlers/context_pipeline.test.ts new file mode 100644 index 000000000000..a02d2416d9fd --- /dev/null +++ b/test/core/crawlers/context_pipeline.test.ts @@ -0,0 +1,167 @@ +import { + ContextPipeline, + ContextPipelineCleanupError, + ContextPipelineInitializationError, + ContextPipelineInterruptedError, + RequestHandlerError, +} from '@crawlee/core'; +import { describe, expect, it, vi } from 'vitest'; + +describe('ContextPipeline', () => { + it('should call middlewares in a sequence', async () => { + const pipeline = ContextPipeline.create() + .compose({ + action: async () => ({ a: 2, b: 1, c: [1] }), + }) + .compose({ + action: async (context) => ({ a: context.a * 2, c: [...context.c, 2] }), + }); + + const consumer = vi.fn(); + await pipeline.call({}, consumer); + + expect(consumer).toHaveBeenCalledWith({ a: 4, b: 1, c: [1, 2] }); + }); + + it('should call cleanup routines', async () => { + const pipeline = ContextPipeline.create() + .compose({ + action: async () => ({ c: [] as number[] }), + cleanup: async (context) => { + context.c.push(1); + }, + }) + .compose({ + action: async () => ({}), + cleanup: async (context) => { + context.c.push(2); + }, + }); + + const consumer = vi.fn(); + await pipeline.call({}, consumer); + + expect(consumer).toHaveBeenCalledWith({ c: [2, 1] }); + }); + + it('should allow interrupting the pipeline in middlewares', async () => { + const context = { a: 3 }; + + const firstAction = vi.fn().mockResolvedValue({}); + const firstCleanup = vi.fn(); + const secondAction = vi.fn().mockRejectedValue(new ContextPipelineInterruptedError()); + const secondCleanup = vi.fn(); + const thirdAction = vi.fn().mockResolvedValue({}); + const thirdCleanup = vi.fn(); + + const pipeline = ContextPipeline.create() + .compose({ action: firstAction, cleanup: firstCleanup }) + .compose({ + action: secondAction, + cleanup: secondCleanup, + }) + .compose({ action: thirdAction, cleanup: thirdCleanup }); + + const consumer = vi.fn(); + + await expect(pipeline.call(context, consumer)).rejects.toThrow(ContextPipelineInterruptedError); + + expect(firstAction).toHaveBeenCalled(); + expect(firstCleanup).toHaveBeenCalled(); + expect(secondAction).toHaveBeenCalled(); + expect(secondCleanup).not.toHaveBeenCalled(); + expect(thirdAction).not.toHaveBeenCalled(); + expect(thirdCleanup).not.toHaveBeenCalled(); + expect(consumer).not.toHaveBeenCalled(); + }); + + it('should wrap pipeline initialization errors', async () => { + const initializationError = new Error('Pipeline initialization failed'); + const context = { a: 3 }; + const secondMiddleware = vi.fn(); + + const pipeline = ContextPipeline.create() + .compose({ + action: async () => { + throw initializationError; + }, + }) + .compose({ action: secondMiddleware }); + + const consumer = vi.fn(); + + await expect(pipeline.call(context, consumer)).rejects.toThrow( + expect.objectContaining({ + cause: initializationError, + constructor: ContextPipelineInitializationError, + }), + ); + + expect(consumer).not.toHaveBeenCalled(); + expect(secondMiddleware).not.toHaveBeenCalled(); + }); + + it('should wrap errors in the final consumer', async () => { + const consumerError = new Error('Request handler failed'); + const context = { a: 3 }; + + const pipeline = ContextPipeline.create().compose({ + action: async () => ({ + b: 4, + }), + }); + + const consumer = vi.fn().mockRejectedValue(consumerError); + + await expect(pipeline.call(context, consumer)).rejects.toThrow( + expect.objectContaining({ + cause: consumerError, + constructor: RequestHandlerError, + }), + ); + + expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 }); + }); + + it('should call cleanup routines even if the final consumer fails', async () => { + const consumerError = new Error('Request handler failed'); + const context = { a: 3 }; + const cleanup = vi.fn(); + + const pipeline = ContextPipeline.create().compose({ + action: async () => ({ + b: 4, + }), + cleanup, + }); + + await expect(pipeline.call(context, vi.fn().mockRejectedValue(consumerError))).rejects.toThrow(); + + expect(cleanup).toHaveBeenCalledWith({ a: 3, b: 4 }, consumerError); + }); + + it('should wrap cleanup errors', async () => { + const cleanupError = new Error('Pipeline cleanup failed'); + const context = { a: 3 }; + + const pipeline = ContextPipeline.create().compose({ + action: async () => ({ + b: 4, + }), + cleanup: async () => { + throw cleanupError; + }, + }); + + const consumer = vi.fn(); + + await expect(pipeline.call(context, consumer)).rejects.toThrow( + expect.objectContaining({ + cause: cleanupError, + constructor: ContextPipelineCleanupError, + }), + ); + + expect(consumer).toHaveBeenCalledWith({ a: 3, b: 4 }); + }); +}); diff --git a/test/core/crawlers/crawler_extension.test.ts b/test/core/crawlers/crawler_extension.test.ts deleted file mode 100644 index 6949953d13b5..000000000000 --- a/test/core/crawlers/crawler_extension.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { CrawlerExtension } from '@crawlee/core'; - -describe('CrawlerExtension', () => { - test('should work', () => { - class MyExtension extends CrawlerExtension {} - const myExtension = new MyExtension(); - expect(myExtension.name).toEqual('MyExtension'); - expect(() => myExtension.getCrawlerOptions()).toThrow( - `${myExtension.name} has not implemented "getCrawlerOptions" method.`, - ); - expect(myExtension.log.info).toBeDefined(); - // @ts-expect-error Accessing private prop - expect(myExtension.log.options.prefix).toEqual('MyExtension'); - }); -}); diff --git a/test/core/crawlers/dom_crawler.test.ts b/test/core/crawlers/dom_crawler.test.ts index 0f027ec816a6..52d4ba8d2f64 100644 --- a/test/core/crawlers/dom_crawler.test.ts +++ b/test/core/crawlers/dom_crawler.test.ts @@ -2,7 +2,7 @@ import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { JSDOMCrawler } from '@crawlee/jsdom'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; const router = new Map(); router.set('/', (req, res) => { diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.ts index 123432ece1b6..83ce50a1ddb7 100644 --- a/test/core/crawlers/file_download.test.ts +++ b/test/core/crawlers/file_download.test.ts @@ -1,24 +1,25 @@ import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; -import { Duplex } from 'node:stream'; +import { Duplex, finished, pipeline as pipelineWithCallbacks, Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { ReadableStream } from 'node:stream/web'; import { setTimeout } from 'node:timers/promises'; -import { Configuration, FileDownload } from '@crawlee/http'; +import { FileDownload } from '@crawlee/http'; import express from 'express'; -import { startExpressAppPromise } from 'test/shared/_helper'; +import { startExpressAppPromise } from 'test/shared/_helper.js'; +import { afterAll, beforeAll, expect, test } from 'vitest'; class ReadableStreamGenerator { - private static async generateRandomData(size: number, seed: number) { + private static async generateRandomData(size: number, seed: number): Promise { const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - const buffer = Buffer.alloc(size); + const array = new Uint8Array(size); for (let i = 0; i < size; i++) { // eslint-disable-next-line no-bitwise seed = Math.imul(48271, seed) | (0 % 2147483647); - buffer[i] = chars.charCodeAt(seed % chars.length); + array[i] = chars.charCodeAt(seed % chars.length); } - return buffer; + return array; } static getReadableStream(size: number, seed: number, throttle = 0): ReadableStream { @@ -42,13 +43,15 @@ class ReadableStreamGenerator { return stream; } - static async getBuffer(size: number, seed: number) { + static async getUint8Array(size: number, seed: number) { const stream = this.getReadableStream(size, seed); - const chunks: string[] = []; + const chunks: Uint8Array = new Uint8Array(size); + let offset = 0; for await (const chunk of stream) { - chunks.push(chunk); + chunks.set(chunk, offset); + offset += chunk.length; } - return Buffer.from(chunks.join('')); + return chunks; } } @@ -80,13 +83,13 @@ afterAll(async () => { server.close(); }); -test('requestHandler works', async () => { - const results: Buffer[] = []; +test('requestHandler - reading bytes synchronously', async () => { + const results: Uint8Array[] = []; const crawler = new FileDownload({ maxRequestRetries: 0, - requestHandler: ({ body }) => { - results.push(body as Buffer); + requestHandler: async ({ response }) => { + results.push(await response.bytes()); }, }); @@ -96,17 +99,17 @@ test('requestHandler works', async () => { expect(results).toHaveLength(1); expect(results[0].length).toBe(1024); - expect(results[0]).toEqual(await ReadableStreamGenerator.getBuffer(1024, 123)); + expect(results[0]).toEqual(await ReadableStreamGenerator.getUint8Array(1024, 123)); }); -test('streamHandler works', async () => { - let result: Buffer = Buffer.alloc(0); +test('requestHandler - streaming response body', async () => { + let result: Uint8Array = new Uint8Array(); const crawler = new FileDownload({ maxRequestRetries: 0, - streamHandler: async ({ stream }) => { - for await (const chunk of stream as unknown as ReadableStream) { - result = Buffer.concat([result, chunk]); + requestHandler: async ({ response }) => { + for await (const chunk of response.body ?? []) { + result = new Uint8Array([...result, ...chunk]); } }, }); @@ -116,18 +119,16 @@ test('streamHandler works', async () => { await crawler.run([fileUrl]); expect(result.length).toBe(1024); - expect(result).toEqual(await ReadableStreamGenerator.getBuffer(1024, 456)); + expect(result).toEqual(await ReadableStreamGenerator.getUint8Array(1024, 456)); }); -test('streamHandler receives response', async () => { +test('requestHandler receives response', async () => { const crawler = new FileDownload({ maxRequestRetries: 0, - streamHandler: async ({ response }) => { - expect(response.headers['content-type']).toBe('application/octet-stream'); - expect(response.rawHeaders[0]).toBe('content-type'); - expect(response.rawHeaders[1]).toBe('application/octet-stream'); - expect(response.statusCode).toBe(200); - expect(response.statusMessage).toBe('OK'); + requestHandler: async ({ response }) => { + expect(response?.headers.get('content-type')).toBe('application/octet-stream'); + expect(response?.status).toBe(200); + expect(response?.statusText).toBe('OK'); }, }); @@ -136,10 +137,10 @@ test('streamHandler receives response', async () => { await crawler.run([fileUrl]); }); -test('crawler with streamHandler waits for the stream to finish', async () => { +test('crawler waits for the stream to be consumed', async () => { const bufferingStream = new Duplex({ read() {}, - write(chunk, encoding, callback) { + write(chunk, _encoding, callback) { this.push(chunk); callback(); }, @@ -147,15 +148,15 @@ test('crawler with streamHandler waits for the stream to finish', async () => { const crawler = new FileDownload({ maxRequestRetries: 0, - streamHandler: ({ stream }) => { - pipeline(stream as any, bufferingStream) - .then(() => { + requestHandler: async ({ response }) => { + pipelineWithCallbacks(response.body ?? ReadableStream.from([]), bufferingStream, (err) => { + if (!err) { bufferingStream.push(null); bufferingStream.end(); - }) - .catch((e) => { - bufferingStream.destroy(e); - }); + } else { + bufferingStream.destroy(err); + } + }); }, }); @@ -166,12 +167,13 @@ test('crawler with streamHandler waits for the stream to finish', async () => { // the stream should be finished once the crawler finishes. expect(bufferingStream.writableFinished).toBe(true); - const bufferedData: Buffer[] = []; + const bufferedData = new Uint8Array(5 * 1024); + let offset = 0; for await (const chunk of bufferingStream) { - bufferedData.push(chunk); + bufferedData.set(chunk, offset); + offset += chunk.length; } - const result = Buffer.concat(bufferedData); - expect(result.length).toBe(5 * 1024); - expect(result).toEqual(await ReadableStreamGenerator.getBuffer(5 * 1024, 789)); + expect(bufferedData.length).toBe(5 * 1024); + expect(bufferedData).toEqual(await ReadableStreamGenerator.getUint8Array(5 * 1024, 789)); }); diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts index d4bf2b0e20b2..3ada1aa0d853 100644 --- a/test/core/crawlers/http_crawler.test.ts +++ b/test/core/crawlers/http_crawler.test.ts @@ -2,9 +2,9 @@ import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { Readable } from 'node:stream'; -import { GotScrapingHttpClient, HttpCrawler } from '@crawlee/http'; -import { ImpitHttpClient } from '@crawlee/impit-client'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { HttpCrawler } from '@crawlee/http'; +import { ResponseWithUrl } from '@crawlee/http-client'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; const router = new Map(); router.set('/', (req, res) => { @@ -95,385 +95,311 @@ afterAll(async () => { await localStorageEmulator.destroy(); }); -describe.each( - process.version.startsWith('v16') - ? [new GotScrapingHttpClient()] - : [new GotScrapingHttpClient(), new ImpitHttpClient()], -)('HttpCrawler with %s', (httpClient) => { - test('works', async () => { - const results: string[] = []; - - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - requestHandler: ({ body }) => { - results.push(body as string); - }, - }); - - await crawler.run([url]); +test('works', async () => { + const results: string[] = []; - expect(results[0].includes('Example Domain')).toBeTruthy(); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: ({ body }) => { + results.push(body as string); + }, }); - test('parseWithCheerio works', async () => { - const results: string[] = []; + await crawler.run([url]); - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - requestHandler: async ({ parseWithCheerio }) => { - const $ = await parseWithCheerio('title'); - results.push($('title').text()); - }, - }); + expect(results[0].includes('Example Domain')).toBeTruthy(); +}); - await crawler.run([`${url}/hello.html`]); +test('parseWithCheerio works', async () => { + const results: string[] = []; - expect(results).toStrictEqual(['Example Domain']); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: async ({ parseWithCheerio }) => { + const $ = await parseWithCheerio('title'); + results.push($('title').text()); + }, }); - test('should parse content type from header', async () => { - const results: { type: string; encoding: BufferEncoding }[] = []; + await crawler.run([`${url}/hello.html`]); - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - requestHandler: ({ contentType }) => { - results.push(contentType); - }, - }); + expect(results).toStrictEqual(['Example Domain']); +}); - await crawler.run([url]); +test('should parse content type from header', async () => { + const results: { type: string; encoding: BufferEncoding }[] = []; - expect(results).toStrictEqual([ - { - type: 'text/html', - encoding: 'utf-8', - }, - ]); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: ({ contentType }) => { + results.push(contentType); + }, }); - test('should parse content type from file extension', async () => { - const results: { type: string; encoding: BufferEncoding }[] = []; + await crawler.run([url]); - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - requestHandler: ({ contentType }) => { - results.push(contentType); - }, - }); + expect(results).toStrictEqual([ + { + type: 'text/html', + encoding: 'utf-8', + }, + ]); +}); - await crawler.run([`${url}/hello.html`]); +test('should parse content type from file extension', async () => { + const results: { type: string; encoding: BufferEncoding }[] = []; - expect(results).toStrictEqual([ - { - type: 'text/html', - encoding: 'utf-8', - }, - ]); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: ({ contentType }) => { + results.push(contentType); + }, }); - test('no content type defaults to octet-stream', async () => { - const results: { type: string; encoding: BufferEncoding }[] = []; + await crawler.run([`${url}/hello.html`]); - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - additionalMimeTypes: ['*/*'], - requestHandler: ({ contentType }) => { - results.push(contentType); - }, - }); + expect(results).toStrictEqual([ + { + type: 'text/html', + encoding: 'utf-8', + }, + ]); +}); - await crawler.run([`${url}/noext`]); +test('no content type defaults to octet-stream', async () => { + const results: { type: string; encoding: BufferEncoding }[] = []; - expect(results).toStrictEqual([ - { - type: 'application/octet-stream', - encoding: 'utf-8', - }, - ]); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + additionalMimeTypes: ['*/*'], + requestHandler: ({ contentType }) => { + results.push(contentType); + }, }); - test('invalid content type defaults to octet-stream', async () => { - const results: { type: string; encoding: BufferEncoding }[] = []; + await crawler.run([`${url}/noext`]); - const crawler = new HttpCrawler({ - httpClient, - maxRequestRetries: 0, - additionalMimeTypes: ['*/*'], - requestHandler: ({ contentType }) => { - results.push(contentType); - }, - }); + expect(results).toStrictEqual([ + { + type: 'application/octet-stream', + encoding: 'utf-8', + }, + ]); +}); - await crawler.run([`${url}/invalidContentType`]); +test('invalid content type defaults to octet-stream', async () => { + const results: { type: string; encoding: BufferEncoding }[] = []; - expect(results).toStrictEqual([ - { - type: 'application/octet-stream', - encoding: 'utf-8', - }, - ]); + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + additionalMimeTypes: ['*/*'], + requestHandler: ({ contentType }) => { + results.push(contentType); + }, }); - test('handles cookies from redirects', async () => { - const results: string[] = []; + await crawler.run([`${url}/invalidContentType`]); - const crawler = new HttpCrawler({ - httpClient, - sessionPoolOptions: { - maxPoolSize: 1, - }, - handlePageFunction: async ({ body }) => { - results.push(JSON.parse(body.toString())); - }, - }); + expect(results).toStrictEqual([ + { + type: 'application/octet-stream', + encoding: 'utf-8', + }, + ]); +}); - await crawler.run([`${url}/redirectAndCookies`]); +test('handles cookies from redirects', async () => { + const results: string[] = []; - expect(results).toStrictEqual(['foo=bar']); + const crawler = new HttpCrawler({ + sessionPoolOptions: { + maxPoolSize: 1, + }, + requestHandler: async ({ body }) => { + results.push(JSON.parse(body.toString())); + }, }); - test('handles cookies from redirects - no empty cookie header', async () => { - const results: string[] = []; + await crawler.run([`${url}/redirectAndCookies`]); - const crawler = new HttpCrawler({ - httpClient, - sessionPoolOptions: { - maxPoolSize: 1, - }, - handlePageFunction: async ({ body }) => { - const str = body.toString(); + expect(results).toStrictEqual(['foo=bar']); +}); - if (str !== '') { - results.push(JSON.parse(str)); - } - }, - }); +test('handles cookies from redirects - no empty cookie header', async () => { + const results: string[] = []; - await crawler.run([`${url}/redirectWithoutCookies`]); + const crawler = new HttpCrawler({ + sessionPoolOptions: { + maxPoolSize: 1, + }, + requestHandler: async ({ body }) => { + const str = body.toString(); - expect(results).toStrictEqual([]); + if (str !== '') { + results.push(JSON.parse(str)); + } + }, }); - test('no empty cookie header', async () => { - const results: string[] = []; + await crawler.run([`${url}/redirectWithoutCookies`]); - const crawler = new HttpCrawler({ - httpClient, - sessionPoolOptions: { - maxPoolSize: 1, - }, - handlePageFunction: async ({ body }) => { - const str = body.toString(); + expect(results).toStrictEqual([]); +}); - if (str !== '') { - results.push(JSON.parse(str)); - } - }, - }); +test('no empty cookie header', async () => { + const results: string[] = []; - await crawler.run([`${url}/cookies`]); + const crawler = new HttpCrawler({ + sessionPoolOptions: { + maxPoolSize: 1, + }, + requestHandler: async ({ body }) => { + const str = body.toString(); - expect(results).toStrictEqual([]); + if (str !== '') { + results.push(JSON.parse(str)); + } + }, }); - test('POST with undefined (empty) payload', async () => { - const results: string[] = []; + await crawler.run([`${url}/cookies`]); - const crawler = new HttpCrawler({ - httpClient, - handlePageFunction: async ({ body }) => { - results.push(body.toString()); - }, - }); + expect(results).toStrictEqual([]); +}); - await crawler.run([ - { - url: `${url}/echo`, - payload: undefined, - method: 'POST', - }, - ]); +test('POST with undefined (empty) payload', async () => { + const results: string[] = []; - expect(results).toStrictEqual(['']); + const crawler = new HttpCrawler({ + requestHandler: async ({ body }) => { + results.push(body.toString()); + }, }); - test('should ignore http error status codes set by user', async () => { - const failed: any[] = []; - - const crawler = new HttpCrawler({ - httpClient, - minConcurrency: 2, - maxConcurrency: 2, - ignoreHttpErrorStatusCodes: [500], - requestHandler: () => {}, - failedRequestHandler: ({ request }) => { - failed.push(request); - }, - }); + await crawler.run([ + { + url: `${url}/echo`, + payload: undefined, + method: 'POST', + }, + ]); - await crawler.run([`${url}/500Error`]); + expect(results).toStrictEqual(['']); +}); - expect(crawler.autoscaledPool!.minConcurrency).toBe(2); - expect(failed).toHaveLength(0); +test('should ignore http error status codes set by user', async () => { + const failed: any[] = []; + + const crawler = new HttpCrawler({ + minConcurrency: 2, + maxConcurrency: 2, + ignoreHttpErrorStatusCodes: [500], + requestHandler: () => {}, + failedRequestHandler: ({ request }) => { + failed.push(request); + }, }); - test('should throw an error on http error status codes set by user', async () => { - const failed: any[] = []; - - const crawler = new HttpCrawler({ - httpClient, - minConcurrency: 2, - maxConcurrency: 2, - additionalHttpErrorStatusCodes: [200], - requestHandler: () => {}, - failedRequestHandler: ({ request }) => { - failed.push(request); - }, - }); + await crawler.run([`${url}/500Error`]); - await crawler.run([`${url}/hello.html`]); + expect(crawler.autoscaledPool!.minConcurrency).toBe(2); + expect(failed).toHaveLength(0); +}); - expect(crawler.autoscaledPool!.minConcurrency).toBe(2); - expect(failed).toHaveLength(1); +test('should throw an error on http error status codes set by user', async () => { + const failed: any[] = []; + + const crawler = new HttpCrawler({ + minConcurrency: 2, + maxConcurrency: 2, + additionalHttpErrorStatusCodes: [200], + requestHandler: () => {}, + failedRequestHandler: ({ request }) => { + failed.push(request); + }, }); - test('should work with delete requests', async () => { - const failed: any[] = []; - - const cheerioCrawler = new HttpCrawler({ - httpClient, - maxConcurrency: 1, - maxRequestRetries: 0, - navigationTimeoutSecs: 5, - requestHandlerTimeoutSecs: 5, - requestHandler: async () => {}, - failedRequestHandler: async ({ request }) => { - failed.push(request); - }, - }); + await crawler.run([`${url}/hello.html`]); - await cheerioCrawler.run([ - { - url: `${url}`, - method: 'DELETE', - }, - ]); + expect(crawler.autoscaledPool!.minConcurrency).toBe(2); + expect(failed).toHaveLength(1); +}); - expect(failed).toHaveLength(0); +test('should work with delete requests', async () => { + const failed: any[] = []; + + const cheerioCrawler = new HttpCrawler({ + maxConcurrency: 1, + maxRequestRetries: 0, + navigationTimeoutSecs: 5, + requestHandlerTimeoutSecs: 5, + requestHandler: async () => {}, + failedRequestHandler: async ({ request }) => { + failed.push(request); + }, }); - test('should retry on 403 even with disallowed content-type', async () => { - const succeeded: any[] = []; - - const crawler = new HttpCrawler({ - httpClient, - maxConcurrency: 1, - maxRequestRetries: 1, - preNavigationHooks: [ - async ({ request }) => { - // mock 403 response with octet stream on first request attempt, but not on - // subsequent retries, so the request should eventually succeed - if (request.retryCount === 0) { - request.url = `${url}/403-with-octet-stream`; - } else { - request.url = url; - } - }, - ], - requestHandler: async ({ request }) => { - succeeded.push(request); - }, - }); - - await crawler.run([url]); + await cheerioCrawler.run([ + { + url: `${url}`, + method: 'DELETE', + }, + ]); - expect(succeeded).toHaveLength(1); - expect(succeeded[0].retryCount).toBe(1); - }); + expect(failed).toHaveLength(0); +}); - test.skipIf(httpClient instanceof ImpitHttpClient)('should work with cacheable-request', async () => { - const isFromCache: Record = {}; - const cache = new Map(); - const crawler = new HttpCrawler({ - httpClient, - maxConcurrency: 1, - preNavigationHooks: [ - async (_, gotOptions) => { - gotOptions.cache = cache; - gotOptions.headers = { - ...gotOptions.headers, - // to force cache - 'cache-control': 'max-stale', - }; - }, - ], - requestHandler: async ({ request, response }) => { - isFromCache[request.uniqueKey] = response.isFromCache; +test('should retry on 403 even with disallowed content-type', async () => { + const succeeded: any[] = []; + + const crawler = new HttpCrawler({ + maxConcurrency: 1, + maxRequestRetries: 1, + preNavigationHooks: [ + async ({ request }) => { + // mock 403 response with octet stream on first request attempt, but not on + // subsequent retries, so the request should eventually succeed + if (request.retryCount === 0) { + request.url = `${url}/403-with-octet-stream`; + } else { + request.url = url; + } }, - }); - await crawler.run([ - { url, uniqueKey: 'first' }, - { url, uniqueKey: 'second' }, - ]); - expect(isFromCache).toEqual({ first: false, second: true }); + ], + requestHandler: async ({ request }) => { + succeeded.push(request); + }, }); - test('works with a custom HttpClient', async () => { - const results: string[] = []; + await crawler.run([url]); - const crawler = new HttpCrawler({ - maxRequestRetries: 0, - requestHandler: async ({ body, sendRequest }) => { - results.push(body as string); + expect(succeeded).toHaveLength(1); + expect(succeeded[0].retryCount).toBe(1); +}); - results.push((await sendRequest()).body); - }, - httpClient: { - async sendRequest(request) { - if (request.responseType !== 'text') { - throw new Error('Not implemented'); - } - - return { - body: 'Hello from sendRequest()' as any, - request, - url, - redirectUrls: [], - statusCode: 200, - headers: {}, - trailers: {}, - complete: true, - }; - }, - async stream(request) { - const stream = new Readable(); - stream.push('Schmexample Domain'); - stream.push(null); - - return { - stream, - downloadProgress: { percent: 100, transferred: 0 }, - uploadProgress: { percent: 100, transferred: 0 }, - request, - url, - redirectUrls: [], - statusCode: 200, - headers: { 'content-type': 'text/html; charset=utf-8' }, - trailers: {}, - complete: true, - }; - }, +test('works with a custom HttpClient', async () => { + const results: string[] = []; + + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: async ({ body, sendRequest }) => { + results.push(body as string); + + results.push(await (await sendRequest()).text()); + }, + httpClient: { + async sendRequest(request) { + return new ResponseWithUrl('Schmexample Domain', { + url: request.url.toString(), + status: 200, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }); }, - }); + }, + }); - await crawler.run([url]); + await crawler.run([url]); - expect(results[0].includes('Schmexample Domain')).toBeTruthy(); - expect(results[1].includes('Hello')).toBeTruthy(); - }); + expect(results[0].includes('Schmexample Domain')).toBeTruthy(); + expect(results[1].includes('Schmexample Domain')).toBeTruthy(); }); diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts index ec65e56976ee..3be8ee69b1d1 100644 --- a/test/core/crawlers/playwright_crawler.test.ts +++ b/test/core/crawlers/playwright_crawler.test.ts @@ -2,24 +2,15 @@ import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import os from 'node:os'; -import type { - Cheerio, - CheerioAPI, - CheerioRoot, - Element, - PlaywrightCrawlingContext, - PlaywrightGotoOptions, - PlaywrightRequestHandler, - Request, -} from '@crawlee/playwright'; +import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, Request } from '@crawlee/playwright'; import { PlaywrightCrawler, RequestList } from '@crawlee/playwright'; import express from 'express'; import playwright from 'playwright'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import log from '@apify/log'; -import { startExpressAppPromise } from '../../shared/_helper'; +import { startExpressAppPromise } from '../../shared/_helper.js'; if (os.platform() === 'win32') vitest.setConfig({ testTimeout: 2 * 60 * 1e3 }); @@ -37,7 +28,7 @@ describe('PlaywrightCrawler', () => { const app = express(); server = await startExpressAppPromise(app, 0); port = (server.address() as AddressInfo).port; - app.get('/', (req, res) => { + app.get('/', (_req, res) => { res.send(`Example Domain`); res.status(200); }); @@ -85,13 +76,8 @@ describe('PlaywrightCrawler', () => { const processed: Request[] = []; const failed: Request[] = []; const requestListLarge = await RequestList.open({ sources: sourcesLarge }); - const requestHandler = async ({ - page, - request, - response, - useState, - }: Parameters[0]) => { - const state = await useState([]); + const requestHandler = async ({ page, request, response, useState }: PlaywrightCrawlingContext) => { + await useState([]); expect(response!.status()).toBe(200); request.userData.title = await page.title(); processed.push(request); @@ -163,66 +149,47 @@ describe('PlaywrightCrawler', () => { expect(Object.keys(options.browserPoolOptions).length).toBe(0); }); - test.each([{ useIncognitoPages: true }, { useIncognitoPages: false }])( - 'should apply launchOptions with useIncognitoPages: $useIncognitoPages', - async ({ useIncognitoPages }) => { - // Some launch options apply to the browser, while some apply to the context. - // Here we use some context options to verify that those are actually applied. - const launchOptions = { - locale: 'cz-CZ', - reducedMotion: 'reduce' as const, - timezoneId: 'Pacific/Tahiti', - }; - - let [timezone, locale, reducedMotion] = ['', '', '']; - - const playwrightCrawler = new PlaywrightCrawler({ - maxConcurrency: 1, - launchContext: { - useIncognitoPages, - launchOptions, - }, - browserPoolOptions: { - // don't overwrite locale with fingerprint's locale - useFingerprints: false, - }, - requestHandler: async ({ page }) => { - [timezone, locale, reducedMotion] = await Promise.all([ - page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone), - page.evaluate(() => navigator.language), - page.evaluate(() => { - return window.matchMedia('(prefers-reduced-motion: reduce)').matches - ? 'reduce' - : 'no-preference'; - }), - ]); - }, - }); - - await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]); - - expect(timezone).toBe(launchOptions.timezoneId); - expect(locale).toBe(launchOptions.locale); - expect(reducedMotion).toBe(launchOptions.reducedMotion); - }, - ); - - test('should have correct types in crawling context', async () => { - const requestHandler = async (crawlingContext: PlaywrightCrawlingContext) => { - // Checking that types are correct - const $ = await crawlingContext.parseWithCheerio(); - - const _cheerioRootType: CheerioRoot = $; - const _apiType: CheerioAPI = $; - const _cheerioElementType: Cheerio = $('div'); + test.each([ + { useIncognitoPages: true }, + { useIncognitoPages: false }, + ])('should apply launchOptions with useIncognitoPages: $useIncognitoPages', async ({ useIncognitoPages }) => { + // Some launch options apply to the browser, while some apply to the context. + // Here we use some context options to verify that those are actually applied. + const launchOptions = { + locale: 'cz-CZ', + reducedMotion: 'reduce' as const, + timezoneId: 'Pacific/Tahiti', }; + let [timezone, locale, reducedMotion] = ['', '', '']; + const playwrightCrawler = new PlaywrightCrawler({ - requestList, - maxRequestRetries: 0, maxConcurrency: 1, - requestHandler, + launchContext: { + useIncognitoPages, + launchOptions, + }, + browserPoolOptions: { + // don't overwrite locale with fingerprint's locale + useFingerprints: false, + }, + requestHandler: async ({ page }) => { + [timezone, locale, reducedMotion] = await Promise.all([ + page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone), + page.evaluate(() => navigator.language), + page.evaluate(() => { + return window.matchMedia('(prefers-reduced-motion: reduce)').matches + ? 'reduce' + : 'no-preference'; + }), + ]); + }, }); - await playwrightCrawler.run(); + + await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]); + + expect(timezone).toBe(launchOptions.timezoneId); + expect(locale).toBe(launchOptions.locale); + expect(reducedMotion).toBe(launchOptions.reducedMotion); }); }); diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts index fc98254a0674..4ab6ccbbba37 100644 --- a/test/core/crawlers/puppeteer_crawler.test.ts +++ b/test/core/crawlers/puppeteer_crawler.test.ts @@ -18,11 +18,11 @@ import { ProxyConfiguration, PuppeteerCrawler, RequestList, RequestQueue, Sessio import type { Cookie } from '@crawlee/types'; import { sleep } from '@crawlee/utils'; import type { Server as ProxyChainServer } from 'proxy-chain'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import log from '@apify/log'; -import { createProxyServer } from '../create-proxy-server'; +import { createProxyServer } from '../create-proxy-server.js'; describe('PuppeteerCrawler', () => { let prevEnvHeadless: string; diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts index 43911773945c..8f985b25609c 100644 --- a/test/core/crawlers/statistics.test.ts +++ b/test/core/crawlers/statistics.test.ts @@ -1,6 +1,6 @@ import { Configuration, EventType, Statistics } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; describe('Statistics', () => { const getPerMinute = (jobCount: number, totalTickMillis: number) => { @@ -34,14 +34,14 @@ describe('Statistics', () => { describe('persist state', () => { // needs to go first for predictability test('should increment id by each new consecutive instance', () => { - expect(stats.id).toEqual(0); + expect(stats.id).toEqual('0'); // @ts-expect-error Accessing private prop expect(Statistics.id).toEqual(1); // @ts-expect-error Accessing private prop expect(stats.persistStateKey).toEqual('SDK_CRAWLER_STATISTICS_0'); const [n1, n2] = [new Statistics(), new Statistics()]; - expect(n1.id).toEqual(1); - expect(n2.id).toEqual(2); + expect(n1.id).toEqual('1'); + expect(n2.id).toEqual('2'); // @ts-expect-error Accessing private prop expect(Statistics.id).toEqual(3); }); @@ -338,4 +338,42 @@ describe('Statistics', () => { expect(stats.state.requestsFinished).toEqual(0); expect(stats.requestRetryHistogram).toEqual([]); }); + + describe('explicit id option', () => { + test('statistics with same explicit id should share persisted state', async () => { + const stats1 = new Statistics({ id: 'shared-stats' }); + stats1.startJob(0); + vitest.advanceTimersByTime(100); + stats1.finishJob(0, 0); + + await stats1.startCapturing(); + await stats1.persistState(); + await stats1.stopCapturing(); + + const stats2 = new Statistics({ id: 'shared-stats' }); + await stats2.startCapturing(); + + expect(stats2.state.requestsFinished).toEqual(1); + + await stats2.stopCapturing(); + }); + + test('statistics with different explicit ids should have isolated state', async () => { + const statsA = new Statistics({ id: 'stats-a' }); + statsA.startJob(0); + vitest.advanceTimersByTime(100); + statsA.finishJob(0, 0); + + await statsA.startCapturing(); + await statsA.persistState(); + await statsA.stopCapturing(); + + const statsB = new Statistics({ id: 'stats-b' }); + await statsB.startCapturing(); + + expect(statsB.state.requestsFinished).toEqual(0); + + await statsB.stopCapturing(); + }); + }); }); diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.ts index 05321bf7894f..b35f451a698a 100644 --- a/test/core/enqueue_links/click_elements.test.ts +++ b/test/core/enqueue_links/click_elements.test.ts @@ -13,7 +13,7 @@ import { } from 'crawlee'; import type { Browser as PWBrowser, Page as PWPage } from 'playwright'; import type { Browser as PPBrowser, Target } from 'puppeteer'; -import { runExampleComServer } from 'test/shared/_helper'; +import { runExampleComServer } from 'test/shared/_helper.js'; function isPuppeteerBrowser(browser: PPBrowser | PWBrowser): browser is PPBrowser { return (browser as PPBrowser).targets !== undefined; diff --git a/test/core/error_tracker.test.ts b/test/core/error_tracker.test.ts index b5e9dcc26057..068a50399fb8 100644 --- a/test/core/error_tracker.test.ts +++ b/test/core/error_tracker.test.ts @@ -1,4 +1,4 @@ -import { ErrorTracker } from '../../packages/core/src/crawlers/error_tracker'; +import { ErrorTracker } from '../../packages/core/src/crawlers/error_tracker.js'; const random = () => Math.random().toString(36).slice(2); diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts index dc038b5c2ff1..b9683a42ee1e 100644 --- a/test/core/playwright_utils.test.ts +++ b/test/core/playwright_utils.test.ts @@ -4,8 +4,8 @@ import path from 'node:path'; import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright'; import type { Browser, Page } from 'playwright'; import { chromium } from 'playwright'; -import { runExampleComServer } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { runExampleComServer } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import log from '@apify/log'; @@ -50,9 +50,13 @@ describe('playwrightUtils', () => { // @ts-expect-error let result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); - await playwrightUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt'), { - surviveNavigations: true, - }); + await playwrightUtils.injectFile( + page, + path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), + { + surviveNavigations: true, + }, + ); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); @@ -75,7 +79,10 @@ describe('playwrightUtils', () => { // @ts-expect-error result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); - await playwrightUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt')); + await playwrightUtils.injectFile( + page, + path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), + ); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); @@ -266,8 +273,8 @@ describe('playwrightUtils', () => { const result = await playwrightUtils.parseWithCheerio(page, true); const text = result('body').text().trim(); - expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(0); - expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); + expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(0); + expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0); }); test('expansion works', async () => { @@ -276,8 +283,8 @@ describe('playwrightUtils', () => { const result = await playwrightUtils.parseWithCheerio(page); const text = result('body').text().trim(); - expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(2); - expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); + expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(2); + expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0); }); }); diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts index e70344109488..ed16bb93c405 100644 --- a/test/core/proxy_configuration.test.ts +++ b/test/core/proxy_configuration.test.ts @@ -1,12 +1,10 @@ import { ProxyConfiguration, Request } from '@crawlee/core'; -const sessionId = 538909250932; - describe('ProxyConfiguration', () => { test('newUrl() should return proxy URL', async () => { const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://proxy.com:1111'] }); expect(proxyConfiguration).toBeInstanceOf(ProxyConfiguration); - expect(await proxyConfiguration.newUrl(sessionId)).toBe('http://proxy.com:1111'); + expect(await proxyConfiguration.newUrl()).toBe('http://proxy.com:1111'); }); test('newProxyInfo() should return ProxyInfo object', async () => { @@ -14,14 +12,13 @@ describe('ProxyConfiguration', () => { const url = 'http://proxy.com:1111'; const proxyInfo = { - sessionId: `${sessionId}`, url, hostname: 'proxy.com', username: '', password: '', port: '1111', }; - expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); + expect(await proxyConfiguration.newProxyInfo()).toEqual(proxyInfo); }); test('newProxyInfo() works with special characters', async () => { @@ -29,14 +26,13 @@ describe('ProxyConfiguration', () => { const proxyConfiguration = new ProxyConfiguration({ proxyUrls: [url] }); const proxyInfo = { - sessionId: `${sessionId}`, url, hostname: 'proxy.com', username: 'user@name', password: 'pass@word', port: '1111', }; - expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); + expect(await proxyConfiguration.newProxyInfo()).toEqual(proxyInfo); }); test('should throw on invalid newUrlFunction', async () => { @@ -140,31 +136,6 @@ describe('ProxyConfiguration', () => { expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[2]); }); - test('should rotate custom URLs with sessions correctly', async () => { - const sessions = ['session_01', 'session_02', 'session_03', 'session_04', 'session_05', 'session_06']; - const proxyConfiguration = new ProxyConfiguration({ - proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'], - }); - - // @ts-expect-error TODO private property? - const proxyUrls = proxyConfiguration.proxyUrls!; - // should use same proxy URL - expect(await proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]); - expect(await proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]); - expect(await proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]); - - // should rotate different proxies - expect(await proxyConfiguration.newUrl(sessions[1])).toEqual(proxyUrls[1]); - expect(await proxyConfiguration.newUrl(sessions[2])).toEqual(proxyUrls[2]); - expect(await proxyConfiguration.newUrl(sessions[3])).toEqual(proxyUrls[0]); - expect(await proxyConfiguration.newUrl(sessions[4])).toEqual(proxyUrls[1]); - expect(await proxyConfiguration.newUrl(sessions[5])).toEqual(proxyUrls[2]); - - // should remember already used session - expect(await proxyConfiguration.newUrl(sessions[1])).toEqual(proxyUrls[1]); - expect(await proxyConfiguration.newUrl(sessions[3])).toEqual(proxyUrls[0]); - }); - test('should throw cannot combine custom methods', async () => { const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333']; const newUrlFunction = () => { @@ -233,16 +204,16 @@ describe('ProxyConfiguration', () => { // @ts-expect-error protected property const tieredProxyUrls = proxyConfiguration.tieredProxyUrls!; - expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[0][0]); - expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[1][0]); - expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[2][0]); + expect(await proxyConfiguration.newUrl({ request })).toEqual(tieredProxyUrls[0][0]); + expect(await proxyConfiguration.newUrl({ request })).toEqual(tieredProxyUrls[1][0]); + expect(await proxyConfiguration.newUrl({ request })).toEqual(tieredProxyUrls[2][0]); // we still get the same (higher) proxy tier even with a new request const request2 = new Request({ url: 'http://example.com/another-resource', }); - expect(await proxyConfiguration.newUrl('session-id', { request: request2 })).toEqual(tieredProxyUrls[2][0]); + expect(await proxyConfiguration.newUrl({ request: request2 })).toEqual(tieredProxyUrls[2][0]); }); test('upshifts and downshifts properly', async () => { @@ -258,7 +229,7 @@ describe('ProxyConfiguration', () => { let gotToTheHighestProxy = false; for (let i = 0; i < 10; i++) { - const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request }); + const lastProxyUrl = await proxyConfiguration.newUrl({ request }); if (lastProxyUrl === tieredProxyUrls[2][0]) { gotToTheHighestProxy = true; break; @@ -271,7 +242,7 @@ describe('ProxyConfiguration', () => { let gotToTheLowestProxy = false; for (let i = 0; i < 20; i++) { - const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request }); + const lastProxyUrl = await proxyConfiguration.newUrl({ request }); if (lastProxyUrl === tieredProxyUrls[0][0]) { gotToTheLowestProxy = true; break; @@ -294,7 +265,7 @@ describe('ProxyConfiguration', () => { let gotToTheHighestProxy = false; for (let i = 0; i < 10; i++) { - const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request: failingRequest }); + const lastProxyUrl = await proxyConfiguration.newUrl({ request: failingRequest }); if (lastProxyUrl === tieredProxyUrls[2][0]) { gotToTheHighestProxy = true; @@ -307,7 +278,7 @@ describe('ProxyConfiguration', () => { let gotToTheLowestProxy = false; for (let i = 0; i < 100; i++) { - const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { + const lastProxyUrl = await proxyConfiguration.newUrl({ request: new Request({ url: `http://example.com/${i}` }), }); diff --git a/test/core/puppeteer_request_interception.test.ts b/test/core/puppeteer_request_interception.test.ts index 352b43e32475..19c2af7b4cea 100644 --- a/test/core/puppeteer_request_interception.test.ts +++ b/test/core/puppeteer_request_interception.test.ts @@ -4,7 +4,7 @@ import { sleep } from '@crawlee/utils'; import { launchPuppeteer, utils } from 'crawlee'; import type { HTTPRequest } from 'puppeteer'; -import { runExampleComServer } from '../shared/_helper'; +import { runExampleComServer } from '../shared/_helper.js'; const { addInterceptRequestHandler, removeInterceptRequestHandler } = utils.puppeteer; diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts index 50c157827a27..d6001a106c86 100644 --- a/test/core/puppeteer_utils.test.ts +++ b/test/core/puppeteer_utils.test.ts @@ -4,8 +4,8 @@ import path from 'node:path'; import { KeyValueStore, launchPuppeteer, puppeteerUtils, Request } from '@crawlee/puppeteer'; import type { Dictionary } from '@crawlee/utils'; import type { Browser, Page, ResponseForRequest } from 'puppeteer'; -import { runExampleComServer } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { runExampleComServer } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import log from '@apify/log'; @@ -51,9 +51,13 @@ describe('puppeteerUtils', () => { // @ts-expect-error let result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); - await puppeteerUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt'), { - surviveNavigations: true, - }); + await puppeteerUtils.injectFile( + page, + path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), + { + surviveNavigations: true, + }, + ); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); @@ -76,7 +80,10 @@ describe('puppeteerUtils', () => { // @ts-expect-error result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); - await puppeteerUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt')); + await puppeteerUtils.injectFile( + page, + path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'), + ); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); @@ -194,24 +201,31 @@ describe('puppeteerUtils', () => { await browser.close(); }); + // TODO verify with others how this behaves test('no expansion with ignoreShadowRoots: true', async () => { const page = await browser.newPage(); await page.goto(`${serverAddress}/special/shadow-root`); const result = await puppeteerUtils.parseWithCheerio(page, true); - const text = result('body').text().trim(); - expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(0); - expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); + + // this is failing on macos + if (process.platform !== 'darwin') { + expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(0); + expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0); + } }); test('expansion works', async () => { const page = await browser.newPage(); await page.goto(`${serverAddress}/special/shadow-root`); const result = await puppeteerUtils.parseWithCheerio(page); - const text = result('body').text().trim(); - expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(2); - expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); + + // this is failing on macos + if (process.platform !== 'darwin') { + expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(2); + expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0); + } }); }); diff --git a/test/core/recoverable_state.test.ts b/test/core/recoverable_state.test.ts index b8a95798b8a1..2a7b76e75f79 100644 --- a/test/core/recoverable_state.test.ts +++ b/test/core/recoverable_state.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; -import { RecoverableState } from '../../packages/core/src/recoverable_state'; -import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator'; +import { RecoverableState } from '../../packages/core/src/recoverable_state.js'; +import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator.js'; interface TestState { counter: number; diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.ts index 0ae3bcc191f7..ba68147b09ca 100644 --- a/test/core/request_list.test.ts +++ b/test/core/request_list.test.ts @@ -7,9 +7,8 @@ import { Request, RequestList, } from '@crawlee/core'; -import type { gotScraping } from '@crawlee/utils'; import { sleep } from '@crawlee/utils'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import { beforeAll, type MockedFunction } from 'vitest'; import log from '@apify/log'; @@ -26,18 +25,24 @@ function shuffle(array: unknown[]): unknown[] { return out; } -vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => { - return { - gotScraping: vitest.fn(), - }; +let mockHttpClient = vitest.mockObject({ + async sendRequest(_request: any, _options?: any) { + return new Response(); + }, + async stream() { + return new Response(); + }, }); -let gotScrapingSpy: MockedFunction; - -beforeAll(async () => { - // @ts-ignore for some reason, this fails when the project is not built :/ - const { gotScraping } = await import('@crawlee/utils'); - gotScrapingSpy = vitest.mocked(gotScraping); +beforeEach(async () => { + mockHttpClient = vitest.mockObject({ + async sendRequest() { + return new Response(); + }, + async stream() { + return new Response(); + }, + }); }); describe('RequestList', () => { @@ -191,9 +196,11 @@ describe('RequestList', () => { test('should use regex parameter to parse urls', async () => { const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd'; const listArr = ['https://example.com', 'HTTP://google.com']; - gotScrapingSpy.mockResolvedValue({ body: listStr } as any); const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g; + + mockHttpClient.sendRequest.mockResolvedValueOnce(new Response(listStr)); + const requestList = await RequestList.open({ sources: [ { @@ -202,12 +209,14 @@ describe('RequestList', () => { regex, }, ], + httpClient: mockHttpClient, }); expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[0] }); expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] }); - expect(gotScrapingSpy).toBeCalledWith({ url: 'http://example.com/list-1', encoding: 'utf8' }); + expect(mockHttpClient.sendRequest).toBeCalled(); + expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); }); test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => { @@ -223,17 +232,18 @@ describe('RequestList', () => { const correctUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv'; - gotScrapingSpy.mockResolvedValue({ body: JSON.stringify(list) } as any); + mockHttpClient.sendRequest.mockImplementation(async () => new Response(list.join('\n'))); const requestList = await RequestList.open({ sources: wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl })), + httpClient: mockHttpClient, }); expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] }); expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] }); expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] }); - expect(gotScrapingSpy).toBeCalledWith({ url: correctUrl, encoding: 'utf8' }); + expect(mockHttpClient.sendRequest.mock.calls[0][0]?.url).toBe(correctUrl); }); test('should handle requestsFromUrl with no URLs', async () => { diff --git a/test/core/request_manager_tandem.test.ts b/test/core/request_manager_tandem.test.ts index 1117de5c6520..0be32ee1d57d 100644 --- a/test/core/request_manager_tandem.test.ts +++ b/test/core/request_manager_tandem.test.ts @@ -1,7 +1,7 @@ import { log, Request, RequestList, RequestManagerTandem, RequestQueue } from '@crawlee/core'; import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest'; -import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator.js'; describe('RequestManagerTandem', () => { let logLevel: number; diff --git a/test/core/serialization.test.ts b/test/core/serialization.test.ts index e81601c3c713..f22faf441af2 100644 --- a/test/core/serialization.test.ts +++ b/test/core/serialization.test.ts @@ -5,7 +5,7 @@ import zlib from 'node:zlib'; import { createDeserialize, deserializeArray, serializeArray } from '@crawlee/core'; -const TEST_JSON_PATH = path.join(__dirname, '..', 'shared', 'data', 'sample.json.gz'); +const TEST_JSON_PATH = path.join(import.meta.dirname, '..', 'shared', 'data', 'sample.json.gz'); const gunzip = util.promisify(zlib.gunzip); diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index a56a97368570..03bb260b916e 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -1,5 +1,5 @@ -import { EVENT_SESSION_RETIRED, ProxyConfiguration, Session, SessionPool } from '@crawlee/core'; -import type { Dictionary } from '@crawlee/utils'; +import { EVENT_SESSION_RETIRED, Session, SessionPool } from '@crawlee/core'; +import { ResponseWithUrl } from '@crawlee/http-client'; import { entries, sleep } from '@crawlee/utils'; import { CookieJar } from 'tough-cookie'; @@ -61,10 +61,12 @@ describe('Session - testing session behaviour ', () => { let error; try { - session.setCookiesFromResponse({ - headers: { Cookie: 'invaldi*{*{*{*-----***@s' }, - url: 'http://localhost:1337', - }); + session.setCookiesFromResponse( + new ResponseWithUrl('', { + headers: { Cookie: 'invaldi*{*{*{*-----***@s' }, + url: 'http://localhost:1337', + }), + ); } catch (e) { error = e; } @@ -148,19 +150,6 @@ describe('Session - testing session behaviour ', () => { }); }); - test('should be valid proxy session', async () => { - const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] }); - session = new Session({ sessionPool }); - let error; - try { - await proxyConfiguration.newUrl(session.id); - } catch (e) { - error = e; - } - - expect(error).toBeUndefined(); - }); - test('should use cookieJar', () => { session = new Session({ sessionPool }); expect(session.cookieJar.setCookie).toBeDefined(); @@ -185,17 +174,6 @@ describe('Session - testing session behaviour ', () => { }); }); - test('should checkStatus work with custom codes', () => { - session = new Session({ sessionPool }); - const customStatusCodes = [100, 202, 300]; - expect(session.retireOnBlockedStatusCodes(100, customStatusCodes)).toBeTruthy(); - expect(session.retireOnBlockedStatusCodes(101, customStatusCodes)).toBeFalsy(); - expect(session.retireOnBlockedStatusCodes(200, customStatusCodes)).toBeFalsy(); - expect(session.retireOnBlockedStatusCodes(202, customStatusCodes)).toBeTruthy(); - expect(session.retireOnBlockedStatusCodes(300, customStatusCodes)).toBeTruthy(); - expect(session.retireOnBlockedStatusCodes(400, customStatusCodes)).toBeFalsy(); - }); - test('setCookies should work', () => { const url = 'https://example.com'; const cookies = [ @@ -304,36 +282,34 @@ describe('Session - testing session behaviour ', () => { describe('.putResponse & .getCookieString', () => { test('should set and update cookies from "set-cookie" header', () => { - const headers: Dictionary = {}; + const headers = new Headers(); + + headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure '); + headers.append('set-cookie', 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT'); - headers['set-cookie'] = [ - 'CSRF=e8b667; Domain=example.com; Secure ', - 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT', - ]; const newSession = new Session({ sessionPool: new SessionPool() }); const url = 'https://example.com'; - newSession.setCookiesFromResponse({ headers, url }); + newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers, url })); let cookies = newSession.getCookieString(url); expect(cookies).toEqual('CSRF=e8b667; id=a3fWa'); const newCookie = 'ABCD=1231231213; Domain=example.com; Secure'; - newSession.setCookiesFromResponse({ headers: { 'set-cookie': newCookie }, url }); + newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers: { 'set-cookie': newCookie }, url })); cookies = newSession.getCookieString(url); expect(cookies).toEqual('CSRF=e8b667; id=a3fWa; ABCD=1231231213'); }); }); test('should correctly persist and init cookieJar', () => { - const headers: Dictionary = {}; + const headers = new Headers(); + + headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure '); + headers.append('set-cookie', 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT'); - headers['set-cookie'] = [ - 'CSRF=e8b667; Domain=example.com; Secure ', - 'id=a3fWa; Expires=Wed, Domain=example.com; 21 Oct 2015 07:28:00 GMT', - ]; const newSession = new Session({ sessionPool: new SessionPool() }); const url = 'https://example.com'; - newSession.setCookiesFromResponse({ headers, url }); + newSession.setCookiesFromResponse(new ResponseWithUrl('', { headers, url })); const old = newSession.getState(); diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts index 7ab17395cc45..5f8af2bba90b 100644 --- a/test/core/session_pool/session_pool.test.ts +++ b/test/core/session_pool/session_pool.test.ts @@ -1,6 +1,6 @@ import { Configuration, EventType, KeyValueStore, Session, SessionPool } from '@crawlee/core'; import { entries } from '@crawlee/utils'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import { Log } from '@apify/log'; diff --git a/test/core/session_pool/session_utils.test.ts b/test/core/session_pool/session_utils.test.ts index aab3f1a98a44..d021c161b00a 100644 --- a/test/core/session_pool/session_utils.test.ts +++ b/test/core/session_pool/session_utils.test.ts @@ -1,41 +1,39 @@ import { getCookiesFromResponse } from '@crawlee/core'; -import type { Dictionary } from '@crawlee/utils'; import { Cookie } from 'tough-cookie'; describe('getCookiesFromResponse', () => { test('should parse cookies if set-cookie is array', () => { - const headers: Dictionary = {}; - const dummyCookies = [ - 'CSRF=e8b667; Domain=example.com; Secure', - 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT', - ]; - headers['set-cookie'] = dummyCookies; - const cookies = getCookiesFromResponse({ headers }); + const headers = new Headers(); + + headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure '); + headers.append('set-cookie', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'); + + const cookies = getCookiesFromResponse(new Response('', { headers })); cookies.forEach((cookie) => { expect(cookie).toBeInstanceOf(Cookie); }); - expect(dummyCookies[0]).toEqual(cookies[0].toString()); - expect(dummyCookies[1]).toEqual(cookies[1].toString()); + expect(cookies[0].toString()).toEqual('CSRF=e8b667; Domain=example.com; Secure'); + expect(cookies[1].toString()).toEqual('id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'); }); test('should parse cookies if set-cookie is string', () => { - const headers: Dictionary = {}; - const dummyCookie = 'CSRF=e8b667; Domain=example.com; Secure'; - headers['set-cookie'] = dummyCookie; - const cookies = getCookiesFromResponse({ headers }); + const headers = new Headers(); + headers.append('set-cookie', 'CSRF=e8b667; Domain=example.com; Secure '); + + const cookies = getCookiesFromResponse(new Response('', { headers })); expect(cookies).toHaveLength(1); - expect(dummyCookie).toEqual(cookies[0].toString()); + expect(cookies[0].toString()).toEqual('CSRF=e8b667; Domain=example.com; Secure'); expect(cookies[0]).toBeInstanceOf(Cookie); }); test('should not throw error on parsing invalid cookie', () => { - const headers: Dictionary = {}; - const dummyCookie = 'totally Invalid Cookie $@$@#$**'; - headers['set-cookie'] = dummyCookie; - const cookies = getCookiesFromResponse({ headers }); + const headers = new Headers(); + headers.append('set-cookie', 'totally Invalid Cookie $@$@#$**'); + + const cookies = getCookiesFromResponse(new Response('', { headers })); expect(cookies).toHaveLength(1); expect(cookies[0]).toBeUndefined(); diff --git a/test/core/sitemap_request_list.test.ts b/test/core/sitemap_request_list.test.ts index 72cc499c90cd..7a752b026cf8 100644 --- a/test/core/sitemap_request_list.test.ts +++ b/test/core/sitemap_request_list.test.ts @@ -6,8 +6,8 @@ import { finished } from 'node:stream/promises'; import { type Request, SitemapRequestList } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import express from 'express'; -import { startExpressAppPromise } from 'test/shared/_helper'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { startExpressAppPromise } from 'test/shared/_helper.js'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; // Express server for serving sitemaps let url = 'http://localhost'; @@ -274,8 +274,8 @@ describe('SitemapRequestList', () => { } expect(list.handledCount()).toBe(2); - expect(list.isFinished()).resolves.toBe(true); - expect(list.fetchNextRequest()).resolves.toBe(null); + await expect(list.isFinished()).resolves.toBe(true); + await expect(list.fetchNextRequest()).resolves.toBe(null); }); test('globs filtering works', async () => { @@ -348,7 +348,7 @@ describe('SitemapRequestList', () => { expect(secondBatch).toHaveLength(5); - expect(list.isFinished()).resolves.toBe(true); + await expect(list.isFinished()).resolves.toBe(true); expect(list.handledCount()).toBe(7); }); @@ -359,7 +359,7 @@ describe('SitemapRequestList', () => { await list.markRequestHandled(request); } - expect(list.isFinished()).resolves.toBe(true); + await expect(list.isFinished()).resolves.toBe(true); expect(list.handledCount()).toBe(7); }); @@ -378,7 +378,7 @@ describe('SitemapRequestList', () => { await list.markRequestHandled(request); } - expect(list.isFinished()).resolves.toBe(true); + await expect(list.isFinished()).resolves.toBe(true); expect(list.isSitemapFullyLoaded()).toBe(false); expect(list.handledCount()).toBe(2); }); @@ -393,7 +393,7 @@ describe('SitemapRequestList', () => { await list.markRequestHandled(request); } - expect(list.isFinished()).resolves.toBe(true); + await expect(list.isFinished()).resolves.toBe(true); expect(list.isSitemapFullyLoaded()).toBe(false); expect(list.handledCount()).toBe(2); }); @@ -410,7 +410,7 @@ describe('SitemapRequestList', () => { await sleep(50); - expect(list.isEmpty()).resolves.toBe(false); + await expect(list.isEmpty()).resolves.toBe(false); await list.persistState(); } @@ -430,8 +430,9 @@ describe('SitemapRequestList', () => { while (!(await list.isFinished())) { const request = await list.fetchNextRequest(); - await list.markRequestHandled(request!); - requests.push(request!); + if (!request) break; + await list.markRequestHandled(request); + requests.push(request); } await expect(list.isEmpty()).resolves.toBe(true); @@ -455,12 +456,13 @@ describe('SitemapRequestList', () => { while (!(await list.isFinished())) { const request = await list.fetchNextRequest(); + if (!request) break; if (counter % 2 === 0) { - await list.markRequestHandled(request!); - requests.push(request!); + await list.markRequestHandled(request); + requests.push(request); } else { - await list.reclaimRequest(request!); + await list.reclaimRequest(request); } counter += 1; @@ -494,7 +496,8 @@ describe('SitemapRequestList', () => { while (!(await newList.isFinished())) { const request = await newList.fetchNextRequest(); - await newList.markRequestHandled(request!); + if (!request) break; + await newList.markRequestHandled(request); } expect(list.handledCount()).toBe(1); diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts index eec10d9d17d9..4007e69119cb 100644 --- a/test/core/storages/dataset.test.ts +++ b/test/core/storages/dataset.test.ts @@ -1,6 +1,6 @@ import { checkAndSerialize, chunkBySize, Configuration, Dataset, KeyValueStore } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts'; diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts index 2476b114a5da..98627dbc8e73 100644 --- a/test/core/storages/key_value_store.test.ts +++ b/test/core/storages/key_value_store.test.ts @@ -2,7 +2,7 @@ import { PassThrough } from 'node:stream'; import { Configuration, KeyValueStore, maybeStringify } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; const localStorageEmulator = new MemoryStorageEmulator(); diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts index 881c4a91bf86..e5bb36286e48 100644 --- a/test/core/storages/request_queue.test.ts +++ b/test/core/storages/request_queue.test.ts @@ -10,24 +10,30 @@ import { RequestQueueV2, STORAGE_CONSISTENCY_DELAY_MILLIS, } from '@crawlee/core'; -import type { gotScraping } from '@crawlee/utils'; import { sleep } from '@crawlee/utils'; +import { gotScraping } from 'got-scraping'; import type { MockedFunction } from 'vitest'; -import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js'; -vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => { - return { - gotScraping: vitest.fn(), - }; +let mockHttpClient = vitest.mockObject({ + async sendRequest(_request: any, _options?: any) { + return new Response(); + }, + async stream() { + return new Response(); + }, }); -let gotScrapingSpy: MockedFunction; - -beforeAll(async () => { - // @ts-ignore for some reason, this fails when the project is not built :/ - const { gotScraping } = await import('@crawlee/utils'); - gotScrapingSpy = vitest.mocked(gotScraping); +beforeEach(async () => { + mockHttpClient = vitest.mockObject({ + async sendRequest() { + return new Response(); + }, + async stream() { + return new Response(); + }, + }); }); describe('RequestQueue remote', () => { @@ -784,10 +790,13 @@ describe('RequestQueue with requestsFromUrl', () => { test('should use regex parameter to parse urls', async () => { const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd'; const listArr = ['https://example.com', 'HTTP://google.com']; - gotScrapingSpy.mockResolvedValue({ body: listStr } as any); + + mockHttpClient.sendRequest.mockResolvedValueOnce(new Response(listStr)); const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g; - const queue = await RequestQueue.open(); + const queue = await RequestQueue.open(null, { + httpClient: mockHttpClient, + }); await queue.addRequest({ method: 'GET', requestsFromUrl: 'http://example.com/list-1', @@ -798,7 +807,8 @@ describe('RequestQueue with requestsFromUrl', () => { expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] }); await queue.drop(); - expect(gotScrapingSpy).toBeCalledWith({ url: 'http://example.com/list-1', encoding: 'utf8' }); + expect(mockHttpClient.sendRequest).toBeCalled(); + expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); }); test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => { @@ -814,16 +824,18 @@ describe('RequestQueue with requestsFromUrl', () => { const correctUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv'; - gotScrapingSpy.mockResolvedValue({ body: JSON.stringify(list) } as any); + mockHttpClient.sendRequest.mockImplementation(async () => new Response(list.join('\n'), { status: 200 })); - const queue = await RequestQueue.open(); + const queue = await RequestQueue.open(null, { + httpClient: mockHttpClient, + }); await queue.addRequests(wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl }))); expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] }); expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] }); expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] }); - expect(gotScrapingSpy).toBeCalledWith({ url: correctUrl, encoding: 'utf8' }); + expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe(correctUrl); await queue.drop(); }); diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.ts index 8a8a41f80f25..84ea9a9801e4 100644 --- a/test/core/storages/utils.test.ts +++ b/test/core/storages/utils.test.ts @@ -1,6 +1,6 @@ import type { Dictionary } from '@crawlee/core'; import { Configuration, KeyValueStore, useState } from '@crawlee/core'; -import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js'; describe('useState', () => { const emulator = new MemoryStorageEmulator(); diff --git a/test/e2e/.eslintrc.json b/test/e2e/.eslintrc.json deleted file mode 100644 index 43153b0c7fdf..000000000000 --- a/test/e2e/.eslintrc.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "root": true, - "extends": ["@apify/eslint-config-ts", "prettier"], - "parserOptions": { - "project": null, - "ecmaVersion": 2022 - }, - "ignorePatterns": ["node_modules", "dist", "**/*.d.ts"], - "rules": { - "@typescript-eslint/ban-ts-comment": 0, - "import/extensions": 0, - "import/no-extraneous-dependencies": 0 - } -} diff --git a/test/e2e/adaptive-playwright-default/test.mjs b/test/e2e/adaptive-playwright-default/test.mjs index 5e6f662e2683..bb3185ebd927 100644 --- a/test/e2e/adaptive-playwright-default/test.mjs +++ b/test/e2e/adaptive-playwright-default/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile index f5f5c882eaca..193a737cc14e 100644 --- a/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile +++ b/test/e2e/adaptive-playwright-robots-file/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/adaptive-playwright-robots-file/actor/package.json b/test/e2e/adaptive-playwright-robots-file/actor/package.json index 144e37179c96..5845b91c72bc 100644 --- a/test/e2e/adaptive-playwright-robots-file/actor/package.json +++ b/test/e2e/adaptive-playwright-robots-file/actor/package.json @@ -4,7 +4,7 @@ "description": "Adaptive Playwright Test - Robots file", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/adaptive-playwright-robots-file/test.mjs b/test/e2e/adaptive-playwright-robots-file/test.mjs index 9edc578f3585..24d4ff294265 100644 --- a/test/e2e/adaptive-playwright-robots-file/test.mjs +++ b/test/e2e/adaptive-playwright-robots-file/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfile index 36afd80b9648..28fbfd65ef4d 100644 --- a/test/e2e/automatic-persist-value/actor/Dockerfile +++ b/test/e2e/automatic-persist-value/actor/Dockerfile @@ -1,8 +1,9 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ +RUN rm -r node_modules RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update --no-audit \ @@ -11,6 +12,7 @@ RUN npm --quiet set progress=false \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ - && npm --version + && npm --version \ + && npm update COPY . ./ diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.json index 1c6c17d01961..b68600ce434e 100644 --- a/test/e2e/automatic-persist-value/actor/package.json +++ b/test/e2e/automatic-persist-value/actor/package.json @@ -4,7 +4,7 @@ "description": "Key-Value Store - Automatic Persist Value Test", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/core": "file:./packages/core", "@crawlee/memory-storage": "file:./packages/memory-storage", @@ -15,6 +15,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjs index 329ac0574f80..ee4cf300d8b3 100644 --- a/test/e2e/automatic-persist-value/test.mjs +++ b/test/e2e/automatic-persist-value/test.mjs @@ -1,4 +1,4 @@ -import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile +++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json index 42a271def376..5df5f6f18ce5 100644 --- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json +++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json @@ -4,7 +4,7 @@ "description": "Autoscaling Pool Test - Max Tasks per Minute", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/core": "file:./packages/core", "@crawlee/memory-storage": "file:./packages/memory-storage", @@ -15,6 +15,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs b/test/e2e/autoscaling-max-tasks-per-minute/test.mjs index 3979c69e0309..1b1182c0cb2c 100644 --- a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs +++ b/test/e2e/autoscaling-max-tasks-per-minute/test.mjs @@ -1,4 +1,4 @@ -import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/camoufox-cloudflare/actor/Dockerfile b/test/e2e/camoufox-cloudflare/actor/Dockerfile index b0215803a48d..ed4c197df80f 100644 --- a/test/e2e/camoufox-cloudflare/actor/Dockerfile +++ b/test/e2e/camoufox-cloudflare/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node-playwright-chrome:20-1.50.1-beta AS builder +FROM apify/actor-node-playwright-chrome:22-beta AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit --ignore-scripts \ && npm update -FROM apify/actor-node-playwright-chrome:20-1.50.1-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.json index b2776bb8e175..70c9a52b91a1 100644 --- a/test/e2e/camoufox-cloudflare/actor/package.json +++ b/test/e2e/camoufox-cloudflare/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Camoufox - Solving Cloudflare Challenge", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -20,6 +20,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/camoufox-cloudflare/test.mjs b/test/e2e/camoufox-cloudflare/test.mjs index 635f6fe27402..867deeeab03f 100644 --- a/test/e2e/camoufox-cloudflare/test.mjs +++ b/test/e2e/camoufox-cloudflare/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs'; if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') { await skipTest('TODO fails to build the docker image now'); diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile index 91fadb14630b..b6068fa63198 100644 --- a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile +++ b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json index 8e788a918600..12b6fbbfcfe0 100644 --- a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json +++ b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - curl-impersonate HTTP client", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -20,6 +20,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/cheerio-curl-impersonate-ts/test.mjs b/test/e2e/cheerio-curl-impersonate-ts/test.mjs index 52bf989d2ec1..48aea4fe78f3 100644 --- a/test/e2e/cheerio-curl-impersonate-ts/test.mjs +++ b/test/e2e/cheerio-curl-impersonate-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); @@ -10,12 +10,14 @@ await expect(datasetItems.length === 1, 'A dataset item was pushed'); const result = datasetItems[0]; -expect(result.body.length > 1000, 'HTML response is not empty'); -expect(result.title.toLowerCase().includes('crawlee'), 'HTML title is correct'); -expect( +await expect(result.body.length > 1000, 'HTML response is not empty'); +await expect(result.title.toLowerCase().includes('crawlee'), 'HTML title is correct'); +await expect( result.userAgent === 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', 'User agent is chrome', ); -expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains client IP'); -expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains client IP'); +await expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains client IP'); +await expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains client IP'); +await expect(result.uuidJsonResponse.uuid !== undefined, 'JSON response contains UUID'); +await expect(JSON.parse(result.uuidTextResponse).uuid !== undefined, 'Text response contains UUID'); diff --git a/test/e2e/cheerio-default-ts/actor/Dockerfile b/test/e2e/cheerio-default-ts/actor/Dockerfile index 59ba4ae8b5e8..943b8d1855ee 100644 --- a/test/e2e/cheerio-default-ts/actor/Dockerfile +++ b/test/e2e/cheerio-default-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.json index d0dfc7875eca..406342a62427 100644 --- a/test/e2e/cheerio-default-ts/actor/package.json +++ b/test/e2e/cheerio-default-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - TypeScript", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -19,6 +19,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/cheerio-default-ts/test.mjs b/test/e2e/cheerio-default-ts/test.mjs index bf2015b4e16e..b843e87e99ec 100644 --- a/test/e2e/cheerio-default-ts/test.mjs +++ b/test/e2e/cheerio-default-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-default/actor/Dockerfile b/test/e2e/cheerio-default/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-default/actor/Dockerfile +++ b/test/e2e/cheerio-default/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-default/actor/package.json b/test/e2e/cheerio-default/actor/package.json index 2f90cefb2057..e3c4442b9a7a 100644 --- a/test/e2e/cheerio-default/actor/package.json +++ b/test/e2e/cheerio-default/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Default", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-default/test.mjs b/test/e2e/cheerio-default/test.mjs index bf2015b4e16e..b843e87e99ec 100644 --- a/test/e2e/cheerio-default/test.mjs +++ b/test/e2e/cheerio-default/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile +++ b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-enqueue-links-base/actor/package.json b/test/e2e/cheerio-enqueue-links-base/actor/package.json index 9c4711b45a0f..15ec65535651 100644 --- a/test/e2e/cheerio-enqueue-links-base/actor/package.json +++ b/test/e2e/cheerio-enqueue-links-base/actor/package.json @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-enqueue-links-base/test.mjs b/test/e2e/cheerio-enqueue-links-base/test.mjs index 502745fdd630..151d89849e25 100644 --- a/test/e2e/cheerio-enqueue-links-base/test.mjs +++ b/test/e2e/cheerio-enqueue-links-base/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-enqueue-links/actor/Dockerfile b/test/e2e/cheerio-enqueue-links/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-enqueue-links/actor/Dockerfile +++ b/test/e2e/cheerio-enqueue-links/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-enqueue-links/actor/package.json b/test/e2e/cheerio-enqueue-links/actor/package.json index cfda48bd8964..df90ff94298f 100644 --- a/test/e2e/cheerio-enqueue-links/actor/package.json +++ b/test/e2e/cheerio-enqueue-links/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Enqueue Links", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-enqueue-links/test.mjs b/test/e2e/cheerio-enqueue-links/test.mjs index d93ac0d4a114..2d0009abc0fa 100644 --- a/test/e2e/cheerio-enqueue-links/test.mjs +++ b/test/e2e/cheerio-enqueue-links/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-error-snapshot/actor/Dockerfile b/test/e2e/cheerio-error-snapshot/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-error-snapshot/actor/Dockerfile +++ b/test/e2e/cheerio-error-snapshot/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-error-snapshot/actor/package.json b/test/e2e/cheerio-error-snapshot/actor/package.json index 988e6e0806c8..05443ecea02c 100644 --- a/test/e2e/cheerio-error-snapshot/actor/package.json +++ b/test/e2e/cheerio-error-snapshot/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Should save errors snapshots", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-error-snapshot/test.mjs b/test/e2e/cheerio-error-snapshot/test.mjs index 912f6a7bf24d..0b857750a2fc 100644 --- a/test/e2e/cheerio-error-snapshot/test.mjs +++ b/test/e2e/cheerio-error-snapshot/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, hasNestedKey } from '../tools.mjs'; +import { expect, getActorTestDir, hasNestedKey, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile +++ b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json index bff7e89fe58c..b29519165857 100644 --- a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json +++ b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Ignore SSL Errors", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-ignore-ssl-errors/test.mjs b/test/e2e/cheerio-ignore-ssl-errors/test.mjs index 235afc5f1717..2325ccba28d5 100644 --- a/test/e2e/cheerio-ignore-ssl-errors/test.mjs +++ b/test/e2e/cheerio-ignore-ssl-errors/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-impit-ts/actor/Dockerfile b/test/e2e/cheerio-impit-ts/actor/Dockerfile index ed192b5e137b..45a644a93aa9 100644 --- a/test/e2e/cheerio-impit-ts/actor/Dockerfile +++ b/test/e2e/cheerio-impit-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/cheerio-impit-ts/actor/package.json b/test/e2e/cheerio-impit-ts/actor/package.json index 03ccac5e739f..ba97a80810a6 100644 --- a/test/e2e/cheerio-impit-ts/actor/package.json +++ b/test/e2e/cheerio-impit-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Impit HTTP client", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -20,6 +20,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/cheerio-impit-ts/test.mjs b/test/e2e/cheerio-impit-ts/test.mjs index 8602dbdb5f0d..218055485b7d 100644 --- a/test/e2e/cheerio-impit-ts/test.mjs +++ b/test/e2e/cheerio-impit-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); @@ -10,8 +10,8 @@ await expect(datasetItems.length === 1, 'A dataset item was pushed'); const result = datasetItems[0]; -expect(result.body.length > 1000, 'HTML response is not empty'); -expect(result.title.toLowerCase().includes('crawlee'), 'HTML title is correct'); -expect(/Gecko\/\d{8} Firefox\/\d{2}/.test(result.userAgent), 'Impit correctly spoofs Firefox'); -expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains client IP'); -expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains client IP'); +await expect(result.body.length > 1000, 'HTML response is not empty'); +await expect(result.title.toLowerCase().includes('crawlee'), 'HTML title is correct'); +await expect(/Gecko\/\d{8} Firefox\/\d{2}/.test(result.userAgent), 'Impit correctly spoofs Firefox'); +await expect(result.clientIpJsonResponse.clientIp !== undefined, 'JSON response contains UUID'); +await expect(JSON.parse(result.clientIpTextResponse).clientIp !== undefined, 'Text response contains UUID'); diff --git a/test/e2e/cheerio-initial-cookies/actor/Dockerfile b/test/e2e/cheerio-initial-cookies/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-initial-cookies/actor/Dockerfile +++ b/test/e2e/cheerio-initial-cookies/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-initial-cookies/actor/package.json b/test/e2e/cheerio-initial-cookies/actor/package.json index 09396b497347..d515793a86b9 100644 --- a/test/e2e/cheerio-initial-cookies/actor/package.json +++ b/test/e2e/cheerio-initial-cookies/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Initial Cookies", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjs index e09a30125dde..136a7d03213b 100644 --- a/test/e2e/cheerio-initial-cookies/test.mjs +++ b/test/e2e/cheerio-initial-cookies/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-max-requests/actor/Dockerfile b/test/e2e/cheerio-max-requests/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-max-requests/actor/Dockerfile +++ b/test/e2e/cheerio-max-requests/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-max-requests/actor/package.json b/test/e2e/cheerio-max-requests/actor/package.json index 454f2a94db6b..e593417b294f 100644 --- a/test/e2e/cheerio-max-requests/actor/package.json +++ b/test/e2e/cheerio-max-requests/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Max Requests Per Crawl", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-max-requests/test.mjs b/test/e2e/cheerio-max-requests/test.mjs index f9faf3d6e1f6..f3b80998fc2d 100644 --- a/test/e2e/cheerio-max-requests/test.mjs +++ b/test/e2e/cheerio-max-requests/test.mjs @@ -1,4 +1,4 @@ -import { initialize, expect, validateDataset, getActorTestDir, runActor } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-page-info/actor/Dockerfile b/test/e2e/cheerio-page-info/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-page-info/actor/Dockerfile +++ b/test/e2e/cheerio-page-info/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-page-info/actor/package.json b/test/e2e/cheerio-page-info/actor/package.json index a3e85e5b8b35..d0fc18b7e438 100644 --- a/test/e2e/cheerio-page-info/actor/package.json +++ b/test/e2e/cheerio-page-info/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Page Info", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-page-info/test.mjs b/test/e2e/cheerio-page-info/test.mjs index 6ed16a4f6b72..db70e11af5a7 100644 --- a/test/e2e/cheerio-page-info/test.mjs +++ b/test/e2e/cheerio-page-info/test.mjs @@ -1,4 +1,4 @@ -import { initialize, expect, validateDataset, getActorTestDir, runActor } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile b/test/e2e/cheerio-request-queue-v2/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-request-queue-v2/actor/Dockerfile +++ b/test/e2e/cheerio-request-queue-v2/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-request-queue-v2/actor/package.json b/test/e2e/cheerio-request-queue-v2/actor/package.json index 59c5f37e61c4..3269ce46ceba 100644 --- a/test/e2e/cheerio-request-queue-v2/actor/package.json +++ b/test/e2e/cheerio-request-queue-v2/actor/package.json @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-request-queue-v2/test.mjs b/test/e2e/cheerio-request-queue-v2/test.mjs index bf2015b4e16e..b843e87e99ec 100644 --- a/test/e2e/cheerio-request-queue-v2/test.mjs +++ b/test/e2e/cheerio-request-queue-v2/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-robots-file/actor/Dockerfile b/test/e2e/cheerio-robots-file/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-robots-file/actor/Dockerfile +++ b/test/e2e/cheerio-robots-file/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-robots-file/actor/package.json b/test/e2e/cheerio-robots-file/actor/package.json index 8751275083d1..fabec5416233 100644 --- a/test/e2e/cheerio-robots-file/actor/package.json +++ b/test/e2e/cheerio-robots-file/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Test - Robots file", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-robots-file/test.mjs b/test/e2e/cheerio-robots-file/test.mjs index a607b32bb974..ee7123ef1479 100644 --- a/test/e2e/cheerio-robots-file/test.mjs +++ b/test/e2e/cheerio-robots-file/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile index 59ba4ae8b5e8..943b8d1855ee 100644 --- a/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile +++ b/test/e2e/cheerio-stop-resume-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/cheerio-stop-resume-ts/actor/package.json b/test/e2e/cheerio-stop-resume-ts/actor/package.json index cf307b836523..59047e938259 100644 --- a/test/e2e/cheerio-stop-resume-ts/actor/package.json +++ b/test/e2e/cheerio-stop-resume-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "Crawler Stop-Resume Test - TypeScript", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -19,6 +19,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/cheerio-stop-resume-ts/test.mjs b/test/e2e/cheerio-stop-resume-ts/test.mjs index b118f15ad612..8beaf8681c80 100644 --- a/test/e2e/cheerio-stop-resume-ts/test.mjs +++ b/test/e2e/cheerio-stop-resume-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile +++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json index 3a0a07ab904a..717c7cfb4e0d 100644 --- a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json +++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json @@ -4,7 +4,7 @@ "description": "Cheerio Crawler Test - Should throw on SSL Errors", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/cheerio-throw-on-ssl-errors/test.mjs b/test/e2e/cheerio-throw-on-ssl-errors/test.mjs index a482ed016752..dcb3d14d92cb 100644 --- a/test/e2e/cheerio-throw-on-ssl-errors/test.mjs +++ b/test/e2e/cheerio-throw-on-ssl-errors/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/input-json5/actor/Dockerfile +++ b/test/e2e/input-json5/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.json index e73dbc423c14..f86996cd5a69 100644 --- a/test/e2e/input-json5/actor/package.json +++ b/test/e2e/input-json5/actor/package.json @@ -4,7 +4,7 @@ "description": "JSON5 input test", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" }, @@ -12,6 +12,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/input-json5/test.mjs b/test/e2e/input-json5/test.mjs index b2444904b5d4..133953b3dc14 100644 --- a/test/e2e/input-json5/test.mjs +++ b/test/e2e/input-json5/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs'; if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') { await skipTest('not supported on platform'); diff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfile index 59ba4ae8b5e8..943b8d1855ee 100644 --- a/test/e2e/jsdom-default-ts/actor/Dockerfile +++ b/test/e2e/jsdom-default-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.json index cefb319689d8..649b1820997f 100644 --- a/test/e2e/jsdom-default-ts/actor/package.json +++ b/test/e2e/jsdom-default-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "JSDOM Crawler Test - TypeScript", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -19,6 +19,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/jsdom-default-ts/test.mjs b/test/e2e/jsdom-default-ts/test.mjs index bf2015b4e16e..b843e87e99ec 100644 --- a/test/e2e/jsdom-default-ts/test.mjs +++ b/test/e2e/jsdom-default-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/jsdom-react-ts/actor/Dockerfile b/test/e2e/jsdom-react-ts/actor/Dockerfile index 59ba4ae8b5e8..943b8d1855ee 100644 --- a/test/e2e/jsdom-react-ts/actor/Dockerfile +++ b/test/e2e/jsdom-react-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/jsdom-react-ts/actor/package.json b/test/e2e/jsdom-react-ts/actor/package.json index b0479560ea63..519a6df912dd 100644 --- a/test/e2e/jsdom-react-ts/actor/package.json +++ b/test/e2e/jsdom-react-ts/actor/package.json @@ -4,7 +4,7 @@ "description": "JSDOM Crawler Test - React - TypeScript", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -19,6 +19,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/jsdom-react-ts/test.mjs b/test/e2e/jsdom-react-ts/test.mjs index 0b89623a5e04..69c2652247ce 100644 --- a/test/e2e/jsdom-react-ts/test.mjs +++ b/test/e2e/jsdom-react-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset, skipTest } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs'; await skipTest('target site no longer exists'); diff --git a/test/e2e/linkedom-default-ts/actor/Dockerfile b/test/e2e/linkedom-default-ts/actor/Dockerfile index 59ba4ae8b5e8..943b8d1855ee 100644 --- a/test/e2e/linkedom-default-ts/actor/Dockerfile +++ b/test/e2e/linkedom-default-ts/actor/Dockerfile @@ -1,5 +1,5 @@ # using multistage build, as we need dev deps to build the TS source code -FROM apify/actor-node:20-beta AS builder +FROM apify/actor-node:22-beta AS builder # copy all files, install all dependencies (including dev deps) and build the project COPY . ./ @@ -7,7 +7,7 @@ RUN npm install --include=dev \ && npm run build # create final image -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta # copy only necessary files COPY --from=builder /usr/src/app/packages ./packages COPY --from=builder /usr/src/app/package.json ./ diff --git a/test/e2e/linkedom-default-ts/actor/package.json b/test/e2e/linkedom-default-ts/actor/package.json index 04796ca89000..d996641fdc87 100644 --- a/test/e2e/linkedom-default-ts/actor/package.json +++ b/test/e2e/linkedom-default-ts/actor/package.json @@ -19,6 +19,9 @@ "@crawlee/core": "file:./packages/core", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "devDependencies": { diff --git a/test/e2e/linkedom-default-ts/test.mjs b/test/e2e/linkedom-default-ts/test.mjs index bf2015b4e16e..b843e87e99ec 100644 --- a/test/e2e/linkedom-default-ts/test.mjs +++ b/test/e2e/linkedom-default-ts/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/migration/actor/Dockerfile b/test/e2e/migration/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/migration/actor/Dockerfile +++ b/test/e2e/migration/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.js index f49dd3be391b..a3ce48163fe0 100644 --- a/test/e2e/migration/actor/main.js +++ b/test/e2e/migration/actor/main.js @@ -1,8 +1,9 @@ -import { Worker, workerData } from 'worker_threads'; -import { URL } from 'url'; -import { once } from 'events'; -import { Actor } from 'apify'; +import { once } from 'node:events'; +import { URL } from 'node:url'; +import { Worker, workerData } from 'node:worker_threads'; + import { CheerioCrawler, Configuration, Dataset } from '@crawlee/cheerio'; +import { Actor } from 'apify'; process.env.CRAWLEE_PURGE_ON_START = '0'; diff --git a/test/e2e/migration/actor/package.json b/test/e2e/migration/actor/package.json index e604cf209efb..76c97f3369d2 100644 --- a/test/e2e/migration/actor/package.json +++ b/test/e2e/migration/actor/package.json @@ -4,7 +4,7 @@ "description": "Migration Test", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/migration/test.mjs b/test/e2e/migration/test.mjs index a60519eea0ff..a806a51b2737 100644 --- a/test/e2e/migration/test.mjs +++ b/test/e2e/migration/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json deleted file mode 100644 index 0be68bf205ad..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "actorSpecification": 1, - "name": "test-playwright-chromium-experimental-containers", - "version": "0.0", - "buildTag": "latest", - "env": null -} diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore b/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore deleted file mode 100644 index ced7cbfc582d..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/actor/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.idea -.DS_Store -node_modules -package-lock.json -apify_storage -crawlee_storage -storage diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile deleted file mode 100644 index 3d3e1b390116..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM node:20 AS builder - -COPY /packages ./packages -COPY /package*.json ./ -RUN npm --quiet set progress=false \ - && npm install --only=prod --no-optional --no-audit \ - && npm update - -FROM apify/actor-node-playwright-chrome:20-beta - -RUN rm -r node_modules -COPY --from=builder /node_modules ./node_modules -COPY --from=builder /packages ./packages -COPY --from=builder /package*.json ./ -COPY /.actor ./.actor -COPY /main.js ./ - -RUN echo "Installed NPM packages:" \ - && (npm list --only=prod --no-optional --all || true) \ - && echo "Node.js version:" \ - && node --version \ - && echo "NPM version:" \ - && npm --version diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/main.js b/test/e2e/playwright-chromium-experimental-containers/actor/main.js deleted file mode 100644 index 887cbb744956..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/actor/main.js +++ /dev/null @@ -1,33 +0,0 @@ -import { Actor } from 'apify'; -import { Dataset, PlaywrightCrawler } from '@crawlee/playwright'; - -// fails after update to playwright 1.29.0, looks like issue the chromium extension, maybe the manifest_version 2 vs 3? -process.exit(404); - -const mainOptions = { - exit: Actor.isAtHome(), - storage: - process.env.STORAGE_IMPLEMENTATION === 'LOCAL' - ? new (await import('@apify/storage-local')).ApifyStorageLocal() - : undefined, -}; - -await Actor.main(async () => { - const crawler = new PlaywrightCrawler({ - proxyConfiguration: await Actor.createProxyConfiguration(), - launchContext: { - experimentalContainers: true, - }, - preNavigationHooks: [ - (_ctx, goToOptions) => { - goToOptions.waitUntil = 'networkidle'; - }, - ], - async requestHandler({ page }) { - const content = await page.content(); - await Dataset.pushData({ ip: content.match(/"clientIp":\s*"(.*)"/)?.[1] }); - }, - }); - - await crawler.run(['https://api.apify.com/v2/browser-info?1', 'https://api.apify.com/v2/browser-info?2']); -}, mainOptions); diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/package.json b/test/e2e/playwright-chromium-experimental-containers/actor/package.json deleted file mode 100644 index 9ea1515b59d1..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/actor/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "test-playwright-chromium-experimental-containers", - "version": "0.0.1", - "description": "Playwright Test - Chromium - Experimental containers", - "dependencies": { - "apify": "next", - "@apify/storage-local": "^2.1.3", - "@crawlee/basic": "file:./packages/basic-crawler", - "@crawlee/browser": "file:./packages/browser-crawler", - "@crawlee/browser-pool": "file:./packages/browser-pool", - "@crawlee/core": "file:./packages/core", - "@crawlee/memory-storage": "file:./packages/memory-storage", - "@crawlee/playwright": "file:./packages/playwright-crawler", - "@crawlee/types": "file:./packages/types", - "@crawlee/utils": "file:./packages/utils", - "playwright": "*" - }, - "overrides": { - "apify": { - "@crawlee/core": "file:./packages/core", - "@crawlee/utils": "file:./packages/utils" - } - }, - "scripts": { - "start": "node main.js" - }, - "type": "module", - "license": "ISC" -} diff --git a/test/e2e/playwright-chromium-experimental-containers/test.mjs b/test/e2e/playwright-chromium-experimental-containers/test.mjs deleted file mode 100644 index ffd167ec7c10..000000000000 --- a/test/e2e/playwright-chromium-experimental-containers/test.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs'; - -await skipTest('on hold'); - -const testActorDirname = getActorTestDir(import.meta.url); -await initialize(testActorDirname); - -const { datasetItems } = await runActor(testActorDirname, 16384); - -await expect(datasetItems.length > 0, 'Has dataset items'); - -const ips = new Set(); - -for (const { ip } of datasetItems) { - await expect(!ips.has(ip), 'Unique proxy ip'); - - ips.add(ip); -} diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/playwright-default/actor/Dockerfile +++ b/test/e2e/playwright-default/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-default/actor/package.json b/test/e2e/playwright-default/actor/package.json index 288a038839ae..fc18843ae2f3 100644 --- a/test/e2e/playwright-default/actor/package.json +++ b/test/e2e/playwright-default/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Default", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-default/test.mjs b/test/e2e/playwright-default/test.mjs index 1bc882da6da8..9aa375ea5340 100644 --- a/test/e2e/playwright-default/test.mjs +++ b/test/e2e/playwright-default/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile +++ b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-enqueue-links-base/actor/package.json b/test/e2e/playwright-enqueue-links-base/actor/package.json index bae23adab47a..d94c327dab96 100644 --- a/test/e2e/playwright-enqueue-links-base/actor/package.json +++ b/test/e2e/playwright-enqueue-links-base/actor/package.json @@ -20,6 +20,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-enqueue-links-base/test.mjs b/test/e2e/playwright-enqueue-links-base/test.mjs index e07a7890a850..e3f25d642317 100644 --- a/test/e2e/playwright-enqueue-links-base/test.mjs +++ b/test/e2e/playwright-enqueue-links-base/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, skipTest } from '../tools.mjs'; await skipTest('too flaky'); diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/playwright-enqueue-links/actor/Dockerfile +++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-enqueue-links/actor/package.json b/test/e2e/playwright-enqueue-links/actor/package.json index 57f57a943adb..c33e2b2be77a 100644 --- a/test/e2e/playwright-enqueue-links/actor/package.json +++ b/test/e2e/playwright-enqueue-links/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Enqueue Links", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -20,6 +20,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-enqueue-links/test.mjs b/test/e2e/playwright-enqueue-links/test.mjs index d088b70d1f32..7dea0d94630c 100644 --- a/test/e2e/playwright-enqueue-links/test.mjs +++ b/test/e2e/playwright-enqueue-links/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json deleted file mode 100644 index d1bf754a588a..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "actorSpecification": 1, - "name": "test-playwright-firefox-experimental-containers", - "version": "0.0", - "buildTag": "latest", - "env": null -} diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/.gitignore b/test/e2e/playwright-firefox-experimental-containers/actor/.gitignore deleted file mode 100644 index ced7cbfc582d..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/actor/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.idea -.DS_Store -node_modules -package-lock.json -apify_storage -crawlee_storage -storage diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile deleted file mode 100644 index a153a02b5b4e..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM node:20 AS builder - -COPY /packages ./packages -COPY /package*.json ./ -RUN npm --quiet set progress=false \ - && npm install --only=prod --no-optional --no-audit \ - && npm update - -FROM apify/actor-node-playwright-firefox:20-beta - -RUN rm -r node_modules -COPY --from=builder /node_modules ./node_modules -COPY --from=builder /packages ./packages -COPY --from=builder /package*.json ./ -COPY /.actor ./.actor -COPY /main.js ./ - -RUN echo "Installed NPM packages:" \ - && (npm list --only=prod --no-optional --all || true) \ - && echo "Node.js version:" \ - && node --version \ - && echo "NPM version:" \ - && npm --version diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/main.js b/test/e2e/playwright-firefox-experimental-containers/actor/main.js deleted file mode 100644 index a07251a8036d..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/actor/main.js +++ /dev/null @@ -1,35 +0,0 @@ -import { Actor } from 'apify'; -import playwright from 'playwright'; -import { Dataset, PlaywrightCrawler } from '@crawlee/playwright'; - -// timeouts nowadays, hard to say why -process.exit(404); - -const mainOptions = { - exit: Actor.isAtHome(), - storage: - process.env.STORAGE_IMPLEMENTATION === 'LOCAL' - ? new (await import('@apify/storage-local')).ApifyStorageLocal() - : undefined, -}; - -await Actor.main(async () => { - const crawler = new PlaywrightCrawler({ - proxyConfiguration: await Actor.createProxyConfiguration(), - launchContext: { - launcher: playwright.firefox, - experimentalContainers: true, - }, - preNavigationHooks: [ - (_ctx, goToOptions) => { - goToOptions.waitUntil = 'networkidle'; - }, - ], - async requestHandler({ page }) { - const content = await page.content(); - await Dataset.pushData({ ip: content.match(/"clientIp":\s*"(.*)"/)?.[1] }); - }, - }); - - await crawler.run(['https://api.apify.com/v2/browser-info?1', 'https://api.apify.com/v2/browser-info?2']); -}, mainOptions); diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/package.json b/test/e2e/playwright-firefox-experimental-containers/actor/package.json deleted file mode 100644 index e8d20f154502..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/actor/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "test-playwright-firefox-experimental-containers", - "version": "0.0.1", - "description": "Playwright Test - Firefox - Experimental containers", - "dependencies": { - "apify": "next", - "@apify/storage-local": "^2.1.3", - "@crawlee/basic": "file:./packages/basic-crawler", - "@crawlee/browser": "file:./packages/browser-crawler", - "@crawlee/browser-pool": "file:./packages/browser-pool", - "@crawlee/core": "file:./packages/core", - "@crawlee/memory-storage": "file:./packages/memory-storage", - "@crawlee/playwright": "file:./packages/playwright-crawler", - "@crawlee/types": "file:./packages/types", - "@crawlee/utils": "file:./packages/utils", - "playwright": "*" - }, - "overrides": { - "apify": { - "@crawlee/core": "file:./packages/core", - "@crawlee/utils": "file:./packages/utils" - } - }, - "scripts": { - "start": "node main.js" - }, - "type": "module", - "license": "ISC" -} diff --git a/test/e2e/playwright-firefox-experimental-containers/test.mjs b/test/e2e/playwright-firefox-experimental-containers/test.mjs deleted file mode 100644 index ffd167ec7c10..000000000000 --- a/test/e2e/playwright-firefox-experimental-containers/test.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs'; - -await skipTest('on hold'); - -const testActorDirname = getActorTestDir(import.meta.url); -await initialize(testActorDirname); - -const { datasetItems } = await runActor(testActorDirname, 16384); - -await expect(datasetItems.length > 0, 'Has dataset items'); - -const ips = new Set(); - -for (const { ip } of datasetItems) { - await expect(!ips.has(ip), 'Unique proxy ip'); - - ips.add(ip); -} diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/playwright-initial-cookies/actor/Dockerfile +++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.json index 266ec86938d3..2f2757f69201 100644 --- a/test/e2e/playwright-initial-cookies/actor/package.json +++ b/test/e2e/playwright-initial-cookies/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Initial Cookies", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-initial-cookies/test.mjs b/test/e2e/playwright-initial-cookies/test.mjs index a24cd3a3ef0e..012966452869 100644 --- a/test/e2e/playwright-initial-cookies/test.mjs +++ b/test/e2e/playwright-initial-cookies/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/playwright-introduction-guide/actor/Dockerfile b/test/e2e/playwright-introduction-guide/actor/Dockerfile index 42d0514ba0a4..d77bdcb02e09 100644 --- a/test/e2e/playwright-introduction-guide/actor/Dockerfile +++ b/test/e2e/playwright-introduction-guide/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-introduction-guide/actor/package.json b/test/e2e/playwright-introduction-guide/actor/package.json index e6e445609a90..496e60f80f2a 100644 --- a/test/e2e/playwright-introduction-guide/actor/package.json +++ b/test/e2e/playwright-introduction-guide/actor/package.json @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-introduction-guide/test.mjs b/test/e2e/playwright-introduction-guide/test.mjs index 6a9573f89263..93a2a16094a3 100644 --- a/test/e2e/playwright-introduction-guide/test.mjs +++ b/test/e2e/playwright-introduction-guide/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/playwright-multi-run/actor/Dockerfile +++ b/test/e2e/playwright-multi-run/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-multi-run/actor/package.json b/test/e2e/playwright-multi-run/actor/package.json index 9f7f2f6ddc56..7ad8ecfe553e 100644 --- a/test/e2e/playwright-multi-run/actor/package.json +++ b/test/e2e/playwright-multi-run/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Multiple run calls to the same crawler", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-multi-run/test.mjs b/test/e2e/playwright-multi-run/test.mjs index 9e23ade4fbd9..55e1c47b05d0 100644 --- a/test/e2e/playwright-multi-run/test.mjs +++ b/test/e2e/playwright-multi-run/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset, skipTest } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs'; if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') { await skipTest('not supported on platform'); diff --git a/test/e2e/playwright-robots-file/actor/Dockerfile b/test/e2e/playwright-robots-file/actor/Dockerfile index f5f5c882eaca..193a737cc14e 100644 --- a/test/e2e/playwright-robots-file/actor/Dockerfile +++ b/test/e2e/playwright-robots-file/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/playwright-robots-file/actor/package.json b/test/e2e/playwright-robots-file/actor/package.json index eabc7e0752ee..5c9865fd332a 100644 --- a/test/e2e/playwright-robots-file/actor/package.json +++ b/test/e2e/playwright-robots-file/actor/package.json @@ -4,7 +4,7 @@ "description": "Playwright Test - Robots file", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/playwright-robots-file/test.mjs b/test/e2e/playwright-robots-file/test.mjs index 3eb38625dc9e..1636b2289253 100644 --- a/test/e2e/playwright-robots-file/test.mjs +++ b/test/e2e/playwright-robots-file/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile index efc72336ddb1..d5925df08b5f 100644 --- a/test/e2e/proxy-rotation/actor/Dockerfile +++ b/test/e2e/proxy-rotation/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/proxy-rotation/actor/package.json b/test/e2e/proxy-rotation/actor/package.json index aa48605818e8..9fed31ba3492 100644 --- a/test/e2e/proxy-rotation/actor/package.json +++ b/test/e2e/proxy-rotation/actor/package.json @@ -4,7 +4,7 @@ "description": "Proxy Test - Rotation", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/proxy-rotation/test.mjs b/test/e2e/proxy-rotation/test.mjs index a7ba42135560..36a82f8ffea1 100644 --- a/test/e2e/proxy-rotation/test.mjs +++ b/test/e2e/proxy-rotation/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile index efc72336ddb1..d5925df08b5f 100644 --- a/test/e2e/puppeteer-default/actor/Dockerfile +++ b/test/e2e/puppeteer-default/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-default/actor/package.json b/test/e2e/puppeteer-default/actor/package.json index 88f43ce9c535..4345341472be 100644 --- a/test/e2e/puppeteer-default/actor/package.json +++ b/test/e2e/puppeteer-default/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Default", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-default/test.mjs b/test/e2e/puppeteer-default/test.mjs index 1bc882da6da8..9aa375ea5340 100644 --- a/test/e2e/puppeteer-default/test.mjs +++ b/test/e2e/puppeteer-default/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile +++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-enqueue-links/actor/package.json b/test/e2e/puppeteer-enqueue-links/actor/package.json index 03c616f31eae..e0c26f8dd11f 100644 --- a/test/e2e/puppeteer-enqueue-links/actor/package.json +++ b/test/e2e/puppeteer-enqueue-links/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Enqueue Links", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -20,6 +20,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-enqueue-links/test.mjs b/test/e2e/puppeteer-enqueue-links/test.mjs index d088b70d1f32..7dea0d94630c 100644 --- a/test/e2e/puppeteer-enqueue-links/test.mjs +++ b/test/e2e/puppeteer-enqueue-links/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile b/test/e2e/puppeteer-error-snapshot/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-error-snapshot/actor/Dockerfile +++ b/test/e2e/puppeteer-error-snapshot/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-error-snapshot/actor/package.json b/test/e2e/puppeteer-error-snapshot/actor/package.json index ce3638b8fd90..a51c01014023 100644 --- a/test/e2e/puppeteer-error-snapshot/actor/package.json +++ b/test/e2e/puppeteer-error-snapshot/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Should save errors snapshots", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-error-snapshot/test.mjs b/test/e2e/puppeteer-error-snapshot/test.mjs index 06207551272c..7306e295d228 100644 --- a/test/e2e/puppeteer-error-snapshot/test.mjs +++ b/test/e2e/puppeteer-error-snapshot/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, hasNestedKey } from '../tools.mjs'; +import { expect, getActorTestDir, hasNestedKey, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile +++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json index 853e41750424..3913bff2c907 100644 --- a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json +++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Ignore SSL Errors", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-ignore-ssl-errors/test.mjs b/test/e2e/puppeteer-ignore-ssl-errors/test.mjs index 500504403f46..c695dfa8a7ea 100644 --- a/test/e2e/puppeteer-ignore-ssl-errors/test.mjs +++ b/test/e2e/puppeteer-ignore-ssl-errors/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile +++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-initial-cookies/actor/package.json b/test/e2e/puppeteer-initial-cookies/actor/package.json index 5244dee8fcd5..6a71eff78282 100644 --- a/test/e2e/puppeteer-initial-cookies/actor/package.json +++ b/test/e2e/puppeteer-initial-cookies/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Initial Cookies", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-initial-cookies/test.mjs b/test/e2e/puppeteer-initial-cookies/test.mjs index a24cd3a3ef0e..012966452869 100644 --- a/test/e2e/puppeteer-initial-cookies/test.mjs +++ b/test/e2e/puppeteer-initial-cookies/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-page-info/actor/Dockerfile +++ b/test/e2e/puppeteer-page-info/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-page-info/actor/package.json b/test/e2e/puppeteer-page-info/actor/package.json index ce29be185dae..adbe59c395bc 100644 --- a/test/e2e/puppeteer-page-info/actor/package.json +++ b/test/e2e/puppeteer-page-info/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Page Info", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-page-info/test.mjs b/test/e2e/puppeteer-page-info/test.mjs index 06d47068cb4b..ed362948ff0f 100644 --- a/test/e2e/puppeteer-page-info/test.mjs +++ b/test/e2e/puppeteer-page-info/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile +++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json index 25efd05127b4..7cc03f0457b6 100644 --- a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json +++ b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Store Pagination with jQuery", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-store-pagination-jquery/test.mjs b/test/e2e/puppeteer-store-pagination-jquery/test.mjs index 8f87841e7009..55dcb1c1fd12 100644 --- a/test/e2e/puppeteer-store-pagination-jquery/test.mjs +++ b/test/e2e/puppeteer-store-pagination-jquery/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile +++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-store-pagination/actor/package.json b/test/e2e/puppeteer-store-pagination/actor/package.json index e02e1950ad87..c5f4681dd3c6 100644 --- a/test/e2e/puppeteer-store-pagination/actor/package.json +++ b/test/e2e/puppeteer-store-pagination/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Store Pagination", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-store-pagination/test.mjs b/test/e2e/puppeteer-store-pagination/test.mjs index 8f87841e7009..55dcb1c1fd12 100644 --- a/test/e2e/puppeteer-store-pagination/test.mjs +++ b/test/e2e/puppeteer-store-pagination/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile index c43460bc59f4..24cb001314d0 100644 --- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile +++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-puppeteer-chrome:20-beta +FROM apify/actor-node-puppeteer-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json index 65b5d8134ab1..6b35e537436a 100644 --- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json +++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json @@ -4,7 +4,7 @@ "description": "Puppeteer Test - Should throw on SSL Errors", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/test.mjs b/test/e2e/puppeteer-throw-on-ssl-errors/test.mjs index 39f6c4d9c1fc..725448fcadae 100644 --- a/test/e2e/puppeteer-throw-on-ssl-errors/test.mjs +++ b/test/e2e/puppeteer-throw-on-ssl-errors/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor, validateDataset } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/request-queue-with-concurrency/actor/Dockerfile b/test/e2e/request-queue-with-concurrency/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/request-queue-with-concurrency/actor/Dockerfile +++ b/test/e2e/request-queue-with-concurrency/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/request-queue-with-concurrency/actor/package.json b/test/e2e/request-queue-with-concurrency/actor/package.json index 381cdb7dbab0..57de6df1f3c6 100644 --- a/test/e2e/request-queue-with-concurrency/actor/package.json +++ b/test/e2e/request-queue-with-concurrency/actor/package.json @@ -4,7 +4,7 @@ "description": "Request Queue Test - Zero Concurrency", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/request-queue-with-concurrency/test.mjs b/test/e2e/request-queue-with-concurrency/test.mjs index 6b1d480435d4..5558860c2242 100644 --- a/test/e2e/request-queue-with-concurrency/test.mjs +++ b/test/e2e/request-queue-with-concurrency/test.mjs @@ -1,7 +1,9 @@ -import { initialize, getActorTestDir, pushActor, startActorOnPlatform, expect } from '../tools.mjs'; +import { setTimeout } from 'node:timers/promises'; + import { Actor } from 'apify'; import { log } from 'crawlee'; -import { setTimeout } from 'node:timers/promises'; + +import { expect, getActorTestDir, initialize, pushActor, startActorOnPlatform } from '../tools.mjs'; if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') { const testActorDirname = getActorTestDir(import.meta.url); diff --git a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile +++ b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/request-queue-zero-concurrency/actor/package.json b/test/e2e/request-queue-zero-concurrency/actor/package.json index 1f24f5ba20d6..a261d2d61d73 100644 --- a/test/e2e/request-queue-zero-concurrency/actor/package.json +++ b/test/e2e/request-queue-zero-concurrency/actor/package.json @@ -4,7 +4,7 @@ "description": "Request Queue Test - Zero Concurrency", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/request-queue-zero-concurrency/test.mjs b/test/e2e/request-queue-zero-concurrency/test.mjs index 42656d0ad0a0..4c3e1eee0a86 100644 --- a/test/e2e/request-queue-zero-concurrency/test.mjs +++ b/test/e2e/request-queue-zero-concurrency/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/request-skip-navigation/actor/Dockerfile b/test/e2e/request-skip-navigation/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/request-skip-navigation/actor/Dockerfile +++ b/test/e2e/request-skip-navigation/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/request-skip-navigation/actor/package.json b/test/e2e/request-skip-navigation/actor/package.json index 07e277b03969..6124a449c702 100644 --- a/test/e2e/request-skip-navigation/actor/package.json +++ b/test/e2e/request-skip-navigation/actor/package.json @@ -4,7 +4,7 @@ "description": "Request Test - skipNavigation", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", "@crawlee/http": "file:./packages/http-crawler", @@ -18,6 +18,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/request-skip-navigation/test.mjs b/test/e2e/request-skip-navigation/test.mjs index 0b518f262b2e..a83abf7cfeda 100644 --- a/test/e2e/request-skip-navigation/test.mjs +++ b/test/e2e/request-skip-navigation/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs index 1fbc005e8929..9f85cd756dd9 100644 --- a/test/e2e/run.mjs +++ b/test/e2e/run.mjs @@ -1,12 +1,11 @@ /* eslint-disable no-loop-func */ import { execSync } from 'node:child_process'; -import { once } from 'node:events'; import { readdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { isMainThread, Worker, workerData } from 'node:worker_threads'; -import { colors, getApifyToken, clearPackages, clearStorage, SKIPPED_TEST_CLOSE_CODE } from './tools.mjs'; +import { clearPackages, clearStorage, colors, getApifyToken, SKIPPED_TEST_CLOSE_CODE } from './tools.mjs'; const basePath = dirname(fileURLToPath(import.meta.url)); @@ -81,7 +80,7 @@ async function run() { `[${dir.name}]`, )} did not call "initialize(import.meta.url)"!`, ); - worker.terminate(); + void worker.terminate(); return; } diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfile index 3d3e1b390116..e079f1c7a563 100644 --- a/test/e2e/session-rotation/actor/Dockerfile +++ b/test/e2e/session-rotation/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS builder +FROM node:22 AS builder COPY /packages ./packages COPY /package*.json ./ @@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional --no-audit \ && npm update -FROM apify/actor-node-playwright-chrome:20-beta +FROM apify/actor-node-playwright-chrome:22-beta RUN rm -r node_modules COPY --from=builder /node_modules ./node_modules diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.json index f34d376ffc52..bdb529e43e7f 100644 --- a/test/e2e/session-rotation/actor/package.json +++ b/test/e2e/session-rotation/actor/package.json @@ -4,7 +4,7 @@ "description": "Session Test - Rotation", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/browser": "file:./packages/browser-crawler", "@crawlee/browser-pool": "file:./packages/browser-pool", @@ -19,6 +19,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/session-rotation/test.mjs b/test/e2e/session-rotation/test.mjs index 5ff4a618c8b4..d6d72e9fff8c 100644 --- a/test/e2e/session-rotation/test.mjs +++ b/test/e2e/session-rotation/test.mjs @@ -1,4 +1,4 @@ -import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; const testActorDirname = getActorTestDir(import.meta.url); await initialize(testActorDirname); diff --git a/test/e2e/storage-open-return-storage-object/actor/Dockerfile b/test/e2e/storage-open-return-storage-object/actor/Dockerfile index 36afd80b9648..f93f444a81fe 100644 --- a/test/e2e/storage-open-return-storage-object/actor/Dockerfile +++ b/test/e2e/storage-open-return-storage-object/actor/Dockerfile @@ -1,4 +1,4 @@ -FROM apify/actor-node:20-beta +FROM apify/actor-node:22-beta COPY packages ./packages COPY package*.json ./ diff --git a/test/e2e/storage-open-return-storage-object/actor/package.json b/test/e2e/storage-open-return-storage-object/actor/package.json index f40826ba029f..78f683047cd0 100644 --- a/test/e2e/storage-open-return-storage-object/actor/package.json +++ b/test/e2e/storage-open-return-storage-object/actor/package.json @@ -4,7 +4,7 @@ "description": "Key-Value Store - Return storage object on open", "dependencies": { "apify": "next", - "@apify/storage-local": "^2.1.3", + "@apify/storage-local": "^2.3.0", "@crawlee/basic": "file:./packages/basic-crawler", "@crawlee/core": "file:./packages/core", "@crawlee/memory-storage": "file:./packages/memory-storage", @@ -15,6 +15,9 @@ "apify": { "@crawlee/core": "file:./packages/core", "@crawlee/utils": "file:./packages/utils" + }, + "@apify/storage-local": { + "better-sqlite3": "^11.10.0" } }, "scripts": { diff --git a/test/e2e/storage-open-return-storage-object/test.mjs b/test/e2e/storage-open-return-storage-object/test.mjs index ed808f24116c..a186160db39c 100644 --- a/test/e2e/storage-open-return-storage-object/test.mjs +++ b/test/e2e/storage-open-return-storage-object/test.mjs @@ -1,4 +1,4 @@ -import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs'; +import { expect, getActorTestDir, initialize, runActor } from '../tools.mjs'; /* This test verifies that the storageObject is correctly returned when the KeyValueStore or Dataset is opened. * The storageObject is the result of the KeyValueStoreClient.get() or Dataset.get() methods, diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index c8fd969bbcdc..38051849f27f 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -6,12 +6,11 @@ import { dirname, join } from 'node:path'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; +import { URL_NO_COMMAS_REGEX } from '@crawlee/utils'; import { Actor } from 'apify'; import fs from 'fs-extra'; import { got } from 'got'; -import { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs'; - /** * @param {string} command * @param {import('node:child_process').ExecSyncOptions} options @@ -191,6 +190,7 @@ export async function runActor(dirName, memory = 4096) { }), ); + // eslint-disable-next-line no-shadow return entries.filter(({ name }) => !isPrivateEntry(name)); } @@ -439,7 +439,7 @@ export async function skipTest(reason) { * @returns {boolean} */ function checkDatasetItem(item, propName) { - if (!item.hasOwnProperty(propName)) { + if (!Object.hasOwn(item, propName)) { return false; } diff --git a/test/shared/MemoryStorageEmulator.ts b/test/shared/MemoryStorageEmulator.ts index c39bb248ec16..a0cad7003907 100644 --- a/test/shared/MemoryStorageEmulator.ts +++ b/test/shared/MemoryStorageEmulator.ts @@ -7,9 +7,9 @@ import { ensureDir } from 'fs-extra'; import log from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; -import { StorageEmulator } from './StorageEmulator'; +import { StorageEmulator } from './StorageEmulator.js'; -const LOCAL_EMULATION_DIR = resolve(__dirname, '..', 'tmp', 'memory-emulation-dir'); +const LOCAL_EMULATION_DIR = resolve(import.meta.dirname, '..', 'tmp', 'memory-emulation-dir'); export class MemoryStorageEmulator extends StorageEmulator { private storage!: MemoryStorage; diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts index 8c275ea9d17e..e1cc1c1cb5cc 100644 --- a/test/shared/_helper.ts +++ b/test/shared/_helper.ts @@ -24,8 +24,8 @@ export const responseSamples = { ' Web Scraping, Data Extraction and Automation · Apify\n' + '\n' + '', - complexXml: fs.readFileSync(path.join(__dirname, 'data/complex.xml'), 'utf-8'), - image: fs.readFileSync(path.join(__dirname, 'data/apify.png')), + complexXml: fs.readFileSync(path.join(import.meta.dirname, 'data/complex.xml'), 'utf-8'), + image: fs.readFileSync(path.join(import.meta.dirname, 'data/apify.png')), html: ` @@ -331,6 +331,17 @@ export async function runExampleComServer(): Promise<[Server, number]> { special.get('/html-entities', (_req, res) => { res.type('html').send('"<>"<>'); }); + + special.get('/set-cookie', (req, res) => { + const cookieName = (req.query.name as string) || 'testCookie'; + const cookieValue = (req.query.value as string) || 'testValue'; + res.setHeader('set-cookie', `${cookieName}=${cookieValue}; Path=/`); + res.type('html').send('Cookie set'); + }); + + special.get('/get-cookies', (req, res) => { + res.json({ cookies: req.headers.cookie || '' }); + }); })(); // "cacheable" site with one page, scripts and stylesheets @@ -349,7 +360,7 @@ export async function runExampleComServer(): Promise<[Server, number]> { app.use('/special', special); app.use('/cacheable', cacheable); - app.get('**/*', async (req, res) => { + app.get('{*splat}', async (req, res) => { await setTimeout(50); res.send(responseSamples.html); }); diff --git a/test/tsconfig.json b/test/tsconfig.json index 7fa113996e27..1202ae509eaa 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -3,21 +3,23 @@ "include": ["**/*", "../packages/*/src/**/*"], "exclude": ["e2e", "**/fixtures/*"], "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", "sourceMap": true, "noUnusedLocals": false, "noUnusedParameters": false, "types": ["vitest/globals"], "paths": { - "crawlee": ["packages/crawlee/src"], - "@crawlee/basic": ["packages/basic-crawler/src"], - "@crawlee/browser": ["packages/browser-crawler/src"], - "@crawlee/http": ["packages/http-crawler/src"], - "@crawlee/linkedom": ["packages/linkedom-crawler/src"], - "@crawlee/jsdom": ["packages/jsdom-crawler/src"], - "@crawlee/cheerio": ["packages/cheerio-crawler/src"], - "@crawlee/playwright": ["packages/playwright-crawler/src"], - "@crawlee/puppeteer": ["packages/puppeteer-crawler/src"], - "@crawlee/*": ["packages/*/src"] + "crawlee": ["packages/crawlee/src/index.ts"], + "@crawlee/basic": ["packages/basic-crawler/src/index.ts"], + "@crawlee/browser": ["packages/browser-crawler/src/index.ts"], + "@crawlee/http": ["packages/http-crawler/src/index.ts"], + "@crawlee/linkedom": ["packages/linkedom-crawler/src/index.ts"], + "@crawlee/jsdom": ["packages/jsdom-crawler/src/index.ts"], + "@crawlee/cheerio": ["packages/cheerio-crawler/src/index.ts"], + "@crawlee/playwright": ["packages/playwright-crawler/src/index.ts"], + "@crawlee/puppeteer": ["packages/puppeteer-crawler/src/index.ts"], + "@crawlee/*": ["packages/*/src/index.ts"] } } } diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.ts index 367119854bbd..b3da6832dc33 100644 --- a/test/utils/cheerio.test.ts +++ b/test/utils/cheerio.test.ts @@ -2,7 +2,7 @@ import type { CheerioRoot } from '@crawlee/utils'; import { htmlToText } from '@crawlee/utils'; import * as cheerio from 'cheerio'; -import * as htmlToTextData from '../shared/data/html_to_text_test_data'; +import * as htmlToTextData from '../shared/data/html_to_text_test_data.js'; const checkHtmlToText = (html: string | CheerioRoot, expectedText: string, hasBody = false) => { const text1 = htmlToText(html); @@ -106,9 +106,9 @@ describe('htmlToText()', () => { test('works with Cheerio object', () => { const html1 = 'Some text'; - checkHtmlToText(cheerio.load(html1, { decodeEntities: true }), 'Some text'); + checkHtmlToText(cheerio.load(html1), 'Some text'); const html2 = '

Text outside of body

'; - checkHtmlToText(cheerio.load(html2, { decodeEntities: true }), 'Text outside of body'); + checkHtmlToText(cheerio.load(html2), 'Text outside of body'); }); }); diff --git a/test/utils/cpu-infoV2.test.ts b/test/utils/cpu-infoV2.test.ts index fbdacd511812..bbecb4c8eeca 100644 --- a/test/utils/cpu-infoV2.test.ts +++ b/test/utils/cpu-infoV2.test.ts @@ -11,7 +11,7 @@ import { getCurrentCpuTicksV2, getSystemCpuUsage, sampleCpuUsage, -} from '../../packages/utils/src/internals/systemInfoV2/cpu-info'; +} from '../../packages/utils/src/internals/system-info/cpu-info.js'; vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => { const original: typeof import('@crawlee/utils') = await importActual(); diff --git a/test/utils/extract-urls.test.ts b/test/utils/extract-urls.test.ts index 99f80a7e065a..6e95d437cedc 100644 --- a/test/utils/extract-urls.test.ts +++ b/test/utils/extract-urls.test.ts @@ -1,15 +1,10 @@ import fs from 'node:fs'; import path from 'node:path'; +import type { BaseHttpClient } from '@crawlee/types'; import { downloadListOfUrls, extractUrls, URL_WITH_COMMAS_REGEX } from '@crawlee/utils'; -vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => { - return { - gotScraping: vitest.fn(), - }; -}); - -const baseDataPath = path.join(__dirname, '..', 'shared', 'data'); +const baseDataPath = path.join(import.meta.dirname, '..', 'shared', 'data'); describe('downloadListOfUrls()', () => { test('downloads a list of URLs', async () => { @@ -19,14 +14,16 @@ describe('downloadListOfUrls()', () => { .split(/[\r\n]+/g) .map((u) => u.trim()); - // @ts-ignore for some reason, this fails when the project is not built :/ - const { gotScraping } = await import('@crawlee/utils'); - const gotScrapingSpy = vitest.mocked(gotScraping); - gotScrapingSpy.mockResolvedValueOnce({ body: text }); + const mockClient: BaseHttpClient = { + async sendRequest() { + return new Response(text); + }, + }; await expect( downloadListOfUrls({ url: 'http://www.nowhere12345.com', + httpClient: mockClient, }), ).resolves.toEqual(arr); }); diff --git a/test/utils/fixtures/parent.js b/test/utils/fixtures/parent.js index 6d0e510cba4a..19e0c7f5bac4 100644 --- a/test/utils/fixtures/parent.js +++ b/test/utils/fixtures/parent.js @@ -1,5 +1,5 @@ -const cp = require('child_process'); +import { exec } from 'node:child_process'; for (let count = 1; count < 10; count++) { - cp.exec('node ./test/utils/fixtures/child.js'); + exec('node ./test/utils/fixtures/child.js'); } diff --git a/test/utils/memory-info.test.ts b/test/utils/memory-info.test.ts deleted file mode 100644 index ffac5a9e2b5b..000000000000 --- a/test/utils/memory-info.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { access, readFile } from 'node:fs/promises'; -import { freemem, totalmem } from 'node:os'; - -import { launchPuppeteer } from '@crawlee/puppeteer'; -import { getMemoryInfo, isDocker } from '@crawlee/utils'; - -vitest.mock('node:os', async (importActual) => { - const originalOs: typeof import('node:os') = await importActual(); - return { - ...originalOs, - freemem: vitest.fn(), - totalmem: vitest.fn(), - }; -}); - -vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => { - const original: typeof import('@crawlee/utils') = await importActual(); - - return { - ...original, - isDocker: vitest.fn(), - }; -}); - -vitest.mock('node:fs/promises', async (importActual) => { - const originalFs: typeof import('node:fs/promises') = await importActual(); - return { - ...originalFs, - readFile: vitest.fn(originalFs.readFile), - access: vitest.fn(originalFs.access), - }; -}); - -const isDockerSpy = vitest.mocked(isDocker); -const freememSpy = vitest.mocked(freemem); -const totalmemSpy = vitest.mocked(totalmem); -const accessSpy = vitest.mocked(access); -// If you use this spy, make sure to reset it to the original implementation at the end of the test. -const readFileSpy = vitest.mocked(readFile); - -describe('getMemoryInfo()', () => { - test('works WITHOUT child process outside the container', async () => { - isDockerSpy.mockResolvedValueOnce(false); - freememSpy.mockReturnValueOnce(222); - totalmemSpy.mockReturnValueOnce(333); - - const data = await getMemoryInfo(); - - expect(freememSpy).toHaveBeenCalled(); - expect(totalmemSpy).toHaveBeenCalled(); - - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - - expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20_000_000); - }); - - test('works WITHOUT child process inside the container', async () => { - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockResolvedValueOnce(); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') { - return Promise.resolve('333'); - } - - if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') { - return Promise.resolve('111'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - const data = await getMemoryInfo(); - - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - - expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20_000_000); - }); - - // TODO: check if this comment is still accurate - // this test hangs because we launch the browser, closing is apparently not enough? - test('works WITH child process outside the container', async () => { - process.env.CRAWLEE_HEADLESS = '1'; - isDockerSpy.mockResolvedValueOnce(false); - freememSpy.mockReturnValueOnce(222); - totalmemSpy.mockReturnValueOnce(333); - - let browser!: Awaited>; - - try { - browser = await launchPuppeteer(); - const data = await getMemoryInfo(); - - expect(freememSpy).toHaveBeenCalled(); - expect(totalmemSpy).toHaveBeenCalled(); - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20_000_000); - expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20_000_000); - } finally { - delete process.env.CRAWLEE_HEADLESS; - await browser?.close(); - } - }); - - // TODO: check if this comment is still accurate - // this test hangs because we launch the browser, closing is apparently not enough? - test('works WITH child process inside the container', async () => { - process.env.CRAWLEE_HEADLESS = '1'; - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockResolvedValueOnce(); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') { - return Promise.resolve('333'); - } - - if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') { - return Promise.resolve('111'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - let browser!: Awaited>; - try { - browser = await launchPuppeteer(); - const data = await getMemoryInfo(); - - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20_000_000); - expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20_000_000); - } finally { - delete process.env.CRAWLEE_HEADLESS; - await browser?.close(); - } - }); - - test('works with cgroup V1 with LIMITED memory', async () => { - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockResolvedValueOnce(); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') { - return Promise.resolve('333'); - } - - if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') { - return Promise.resolve('111'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - const data = await getMemoryInfo(); - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - }); - - test('works with cgroup V1 with UNLIMITED memory', async () => { - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockResolvedValueOnce(); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') { - return Promise.resolve('9223372036854771712'); - } - - if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') { - return Promise.resolve('111'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - totalmemSpy.mockReturnValueOnce(333); - - const data = await getMemoryInfo(); - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - }); - - test('works with cgroup V2 with LIMITED memory', async () => { - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockRejectedValueOnce(new Error('ENOENT')); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory.max') { - return Promise.resolve('333\n'); - } - - if (path === '/sys/fs/cgroup/memory.current') { - return Promise.resolve('111\n'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - const data = await getMemoryInfo(); - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - }); - - test('works with cgroup V2 with UNLIMITED memory', async () => { - isDockerSpy.mockResolvedValueOnce(true); - accessSpy.mockRejectedValueOnce(new Error('ENOENT')); - - readFileSpy.mockImplementation(async (path) => { - if (path === '/sys/fs/cgroup/memory.max') { - return Promise.resolve('max\n'); - } - - if (path === '/sys/fs/cgroup/memory.current') { - return Promise.resolve('111\n'); - } - - throw new Error(`Unexpected path ${path}`); - }); - - totalmemSpy.mockReturnValueOnce(333); - - const data = await getMemoryInfo(); - expect(data).toMatchObject({ - totalBytes: 333, - freeBytes: 222, - usedBytes: 111, - }); - }); -}); diff --git a/test/utils/memory-infoV2.test.ts b/test/utils/memory-infoV2.test.ts index df5e9b4585ee..1623336a9ec9 100644 --- a/test/utils/memory-infoV2.test.ts +++ b/test/utils/memory-infoV2.test.ts @@ -2,7 +2,7 @@ import { access, readFile } from 'node:fs/promises'; import { freemem, totalmem } from 'node:os'; import { launchPuppeteer } from '@crawlee/puppeteer'; -import { getCgroupsVersion, getMemoryInfoV2 } from '@crawlee/utils'; +import { getCgroupsVersion, getMemoryInfo } from '@crawlee/utils'; vitest.mock('node:os', async (importActual) => { const originalOs: typeof import('node:os') = await importActual(); @@ -43,7 +43,7 @@ describe('getMemoryInfoV2()', () => { freememSpy.mockReturnValueOnce(222); totalmemSpy.mockReturnValueOnce(333); - const data = await getMemoryInfoV2(); + const data = await getMemoryInfo(); expect(freememSpy).toHaveBeenCalled(); expect(totalmemSpy).toHaveBeenCalled(); @@ -73,7 +73,7 @@ describe('getMemoryInfoV2()', () => { throw new Error(`Unexpected path ${path}`); }); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, @@ -93,7 +93,7 @@ describe('getMemoryInfoV2()', () => { try { browser = await launchPuppeteer(); - const data = await getMemoryInfoV2(); + const data = await getMemoryInfo(); expect(freememSpy).toHaveBeenCalled(); expect(totalmemSpy).toHaveBeenCalled(); @@ -130,7 +130,7 @@ describe('getMemoryInfoV2()', () => { let browser!: Awaited>; try { browser = await launchPuppeteer(); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, @@ -161,7 +161,7 @@ describe('getMemoryInfoV2()', () => { throw new Error(`Unexpected path ${path}`); }); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, freeBytes: 222, @@ -187,7 +187,7 @@ describe('getMemoryInfoV2()', () => { totalmemSpy.mockReturnValueOnce(333); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, freeBytes: 222, @@ -211,7 +211,7 @@ describe('getMemoryInfoV2()', () => { throw new Error(`Unexpected path ${path}`); }); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, freeBytes: 222, @@ -237,7 +237,7 @@ describe('getMemoryInfoV2()', () => { totalmemSpy.mockReturnValueOnce(333); - const data = await getMemoryInfoV2(true); + const data = await getMemoryInfo(true); expect(data).toMatchObject({ totalBytes: 333, freeBytes: 222, diff --git a/test/utils/psTree.test.ts b/test/utils/psTree.test.ts index 8c7f3079a3aa..988ed2f5d3d0 100644 --- a/test/utils/psTree.test.ts +++ b/test/utils/psTree.test.ts @@ -1,11 +1,11 @@ import { exec } from 'node:child_process'; import path from 'node:path'; -import { psTree } from '../../packages/utils/src/internals/systemInfoV2/ps-tree'; +import { psTree } from '../../packages/utils/src/internals/system-info/ps-tree.js'; const scripts = { - parent: path.join(__dirname, 'fixtures', 'parent.js'), - child: path.join(__dirname, 'fixtures', 'child.js'), + parent: path.join(import.meta.dirname, 'fixtures', 'parent.js'), + child: path.join(import.meta.dirname, 'fixtures', 'child.js'), }; describe('psTree()', () => { diff --git a/tsconfig.build.json b/tsconfig.build.json index a60757218988..95710e031b70 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,16 +1,15 @@ { "extends": "@apify/tsconfig", "compilerOptions": { - "target": "ES2020", - "lib": ["ESNext", "DOM", "ES2020"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ESNext", + "lib": ["DOM", "ES2023", "ES2024", "DOM.AsyncIterable"], "baseUrl": ".", "allowJs": true, "skipLibCheck": true, "resolveJsonModule": false, - "emitDecoratorMetadata": false, - "module": "Node16", - "moduleResolution": "Node16" + "emitDecoratorMetadata": false }, - "include": ["./packages/*/src/**/*"], "exclude": ["**/node_modules", "**/dist"] } diff --git a/tsconfig.json b/tsconfig.json index c2c65813fff7..57c09353ca6e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,17 +2,20 @@ "extends": "./tsconfig.build.json", "compilerOptions": { "baseUrl": ".", + "noErrorTruncation": true, + "sourceMap": true, + "declaration": true, "paths": { - "crawlee": ["packages/crawlee/src"], - "@crawlee/basic": ["packages/basic-crawler/src"], - "@crawlee/browser": ["packages/browser-crawler/src"], - "@crawlee/http": ["packages/http-crawler/src"], - "@crawlee/linkedom": ["packages/linkedom-crawler/src"], - "@crawlee/jsdom": ["packages/jsdom-crawler/src"], - "@crawlee/cheerio": ["packages/cheerio-crawler/src"], - "@crawlee/playwright": ["packages/playwright-crawler/src"], - "@crawlee/puppeteer": ["packages/puppeteer-crawler/src"], - "@crawlee/*": ["packages/*/src"] + "crawlee": ["packages/crawlee/src/index.ts"], + "@crawlee/basic": ["packages/basic-crawler/src/index.ts"], + "@crawlee/browser": ["packages/browser-crawler/src/index.ts"], + "@crawlee/http": ["packages/http-crawler/src/index.ts"], + "@crawlee/linkedom": ["packages/linkedom-crawler/src/index.ts"], + "@crawlee/jsdom": ["packages/jsdom-crawler/src/index.ts"], + "@crawlee/cheerio": ["packages/cheerio-crawler/src/index.ts"], + "@crawlee/playwright": ["packages/playwright-crawler/src/index.ts"], + "@crawlee/puppeteer": ["packages/puppeteer-crawler/src/index.ts"], + "@crawlee/*": ["packages/*/src/index.ts"] } } } diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 9246c6ec18ab..9b6f457226bc 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -8,6 +8,7 @@ const packages = [ 'basic-crawler', 'browser-crawler', 'http-crawler', + 'http-client', 'cheerio-crawler', 'puppeteer-crawler', 'playwright-crawler', @@ -26,6 +27,7 @@ const packagesOrder = [ '@crawlee/linkedom', '@crawlee/basic', '@crawlee/http', + '@crawlee/http-client', '@crawlee/browser', '@crawlee/memory-storage', '@crawlee/browser-pool', diff --git a/website/src/components/ApiLink.jsx b/website/src/components/ApiLink.jsx index 947584c85f7b..ad548fd8fce7 100644 --- a/website/src/components/ApiLink.jsx +++ b/website/src/components/ApiLink.jsx @@ -4,10 +4,10 @@ import Link from '@docusaurus/Link'; import { useDocsVersion } from '@docusaurus/plugin-content-docs/client'; import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; -const pkg = require('../../../packages/crawlee/package.json'); +const { version: packageJsonVersion } = require('../../../packages/crawlee/package.json'); -const [v1, v2] = pkg.version.split('.'); -const stable = [v1, v2].join('.'); +const [major, minor] = packageJsonVersion.split('.'); +const stable = [major, minor].join('.'); const ApiLink = ({ to, children }) => { const version = useDocsVersion(); diff --git a/website/versioned_docs/version-4.0/api-packages.json b/website/versioned_docs/version-4.0/api-packages.json new file mode 100644 index 000000000000..09183490975c --- /dev/null +++ b/website/versioned_docs/version-4.0/api-packages.json @@ -0,0 +1 @@ +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/core","packagePath":"packages/core","packageSlug":"core","packageName":"@crawlee/core","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/browser-pool","packagePath":"packages/browser-pool","packageSlug":"browser-pool","packageName":"@crawlee/browser-pool","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/basic-crawler","packagePath":"packages/basic-crawler","packageSlug":"basic-crawler","packageName":"@crawlee/basic","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/browser-crawler","packagePath":"packages/browser-crawler","packageSlug":"browser-crawler","packageName":"@crawlee/browser","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/http-crawler","packagePath":"packages/http-crawler","packageSlug":"http-crawler","packageName":"@crawlee/http","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/http-client","packagePath":"packages/http-client","packageSlug":"http-client","packageName":"@crawlee/http-client","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/cheerio-crawler","packagePath":"packages/cheerio-crawler","packageSlug":"cheerio-crawler","packageName":"@crawlee/cheerio","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/puppeteer-crawler","packagePath":"packages/puppeteer-crawler","packageSlug":"puppeteer-crawler","packageName":"@crawlee/puppeteer","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/playwright-crawler","packagePath":"packages/playwright-crawler","packageSlug":"playwright-crawler","packageName":"@crawlee/playwright","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/jsdom-crawler","packagePath":"packages/jsdom-crawler","packageSlug":"jsdom-crawler","packageName":"@crawlee/jsdom","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/linkedom-crawler","packagePath":"packages/linkedom-crawler","packageSlug":"linkedom-crawler","packageName":"@crawlee/linkedom","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/memory-storage","packagePath":"packages/memory-storage","packageSlug":"memory-storage","packageName":"@crawlee/memory-storage","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/utils","packagePath":"packages/utils","packageSlug":"utils","packageName":"@crawlee/utils","packageVersion":"4.0.0"},{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":"../packages/types","packagePath":"packages/types","packageSlug":"types","packageName":"@crawlee/types","packageVersion":"4.0.0"}] \ No newline at end of file diff --git a/website/versioned_docs/version-4.0/api-typedoc.json b/website/versioned_docs/version-4.0/api-typedoc.json new file mode 100644 index 000000000000..78135bdc006a --- /dev/null +++ b/website/versioned_docs/version-4.0/api-typedoc.json @@ -0,0 +1,396590 @@ +{ + "id": 0, + "name": "@crawlee/root", + "variant": "project", + "kind": 1, + "flags": {}, + "children": [ + { + "id": 3, + "name": "basic-crawler/src", + "variant": "declaration", + "kind": 2, + "flags": {}, + "children": [ + { + "id": 7801, + "name": "AddRequestsBatchedOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 976, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L976" + } + ], + "target": 3319 + }, + { + "id": 7802, + "name": "AddRequestsBatchedResult", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 994, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L994" + } + ], + "target": 3325 + }, + { + "id": 7671, + "name": "AutoscaledPool", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/autoscaled_pool.ts", + "line": 180, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/autoscaled_pool.ts#L180" + } + ], + "target": 267 + }, + { + "id": 7670, + "name": "AutoscaledPoolOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/autoscaled_pool.ts", + "line": 16, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/autoscaled_pool.ts#L16" + } + ], + "target": 243 + }, + { + "id": 7760, + "name": "BLOCKED_STATUS_CODES", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/consts.ts", + "line": 1, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/consts.ts#L1" + } + ], + "target": 2507 + }, + { + "id": 7815, + "name": "checkStorageAccess", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/access_checking.ts", + "line": 10, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/access_checking.ts#L10" + } + ], + "target": 3429 + }, + { + "id": 7676, + "name": "ClientInfo", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/system_status.ts", + "line": 79, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/system_status.ts#L79" + } + ], + "target": 461 + }, + { + "id": 7680, + "name": "Configuration", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/configuration.ts", + "line": 241, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/configuration.ts#L241" + } + ], + "target": 556 + }, + { + "id": 7679, + "name": "ConfigurationOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/configuration.ts", + "line": 18, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/configuration.ts#L18" + } + ], + "target": 513 + }, + { + "id": 7681, + "name": "ContextMiddleware", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/context_pipeline.ts", + "line": 17, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/context_pipeline.ts#L17" + } + ], + "target": 657 + }, + { + "id": 7682, + "name": "ContextPipeline", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/context_pipeline.ts", + "line": 34, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/context_pipeline.ts#L34" + } + ], + "target": 669 + }, + { + "id": 7668, + "name": "ContextPipelineCleanupError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 51, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L51" + } + ], + "target": 207 + }, + { + "id": 7667, + "name": "ContextPipelineInitializationError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 45, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L45" + } + ], + "target": 189 + }, + { + "id": 7666, + "name": "ContextPipelineInterruptedError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 39, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L39" + } + ], + "target": 172 + }, + { + "id": 7659, + "name": "Cookie", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/index.ts", + "line": 18, + "character": 60, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/index.ts#L18" + } + ], + "target": 60 + }, + { + "id": 7688, + "name": "CrawlingContext", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 109, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L109" + } + ], + "target": 753 + }, + { + "id": 7757, + "name": "CreateSession", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/session_pool.ts", + "line": 22, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/session_pool.ts#L22" + } + ], + "target": 2278 + }, + { + "id": 7662, + "name": "CriticalError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 10, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L10" + } + ], + "target": 98 + }, + { + "id": 7773, + "name": "Dataset", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 232, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L232" + } + ], + "target": 2769 + }, + { + "id": 7774, + "name": "DatasetConsumer", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 703, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L703" + } + ], + "target": 2851 + }, + { + "id": 7778, + "name": "DatasetContent", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 742, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L742" + } + ], + "target": 2874 + }, + { + "id": 7769, + "name": "DatasetDataOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 92, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L92" + } + ], + "target": 2732 + }, + { + "id": 7770, + "name": "DatasetExportOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 144, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L144" + } + ], + "target": 2741 + }, + { + "id": 7772, + "name": "DatasetExportToOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 176, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L176" + } + ], + "target": 2759 + }, + { + "id": 7771, + "name": "DatasetIteratorOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 152, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L152" + } + ], + "target": 2749 + }, + { + "id": 7775, + "name": "DatasetMapper", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 714, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L714" + } + ], + "target": 2856 + }, + { + "id": 7777, + "name": "DatasetOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 735, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L735" + } + ], + "target": 2869 + }, + { + "id": 7776, + "name": "DatasetReducer", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/dataset.ts", + "line": 726, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/dataset.ts#L726" + } + ], + "target": 2862 + }, + { + "id": 7701, + "name": "enqueueLinks", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 274, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L274" + } + ], + "target": 1095 + }, + { + "id": 7703, + "name": "EnqueueLinksOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 34, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L34" + } + ], + "target": 1128 + }, + { + "id": 7704, + "name": "EnqueueStrategy", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 216, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L216" + } + ], + "target": 1148 + }, + { + "id": 7696, + "name": "ErrnoException", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/error_tracker.ts", + "line": 10, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/error_tracker.ts#L10" + } + ], + "target": 1021 + }, + { + "id": 7700, + "name": "ErrorSnapshotter", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/error_snapshotter.ts", + "line": 39, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/error_snapshotter.ts#L39" + } + ], + "target": 1071 + }, + { + "id": 7698, + "name": "ErrorTracker", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/error_tracker.ts", + "line": 287, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/error_tracker.ts#L287" + } + ], + "target": 1038 + }, + { + "id": 7697, + "name": "ErrorTrackerOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/error_tracker.ts", + "line": 18, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/error_tracker.ts#L18" + } + ], + "target": 1030 + }, + { + "id": 7727, + "name": "EventManager", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/events/event_manager.ts", + "line": 24, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/events/event_manager.ts#L24" + } + ], + "target": 1234 + }, + { + "id": 7725, + "name": "EventType", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/events/event_manager.ts", + "line": 9, + "character": 18, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/events/event_manager.ts#L9" + } + ], + "target": 1227 + }, + { + "id": 7726, + "name": "EventTypeName", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/events/event_manager.ts", + "line": 17, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/events/event_manager.ts#L17" + } + ], + "target": 1233 + }, + { + "id": 7712, + "name": "filterRequestsByPatterns", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 214, + "character": 16, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L214" + } + ], + "target": 1184 + }, + { + "id": 7677, + "name": "FinalStatistics", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/system_status.ts", + "line": 85, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/system_status.ts#L85" + } + ], + "target": 465 + }, + { + "id": 7747, + "name": "GetUserDataFromRequest", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/router.ts", + "line": 15, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/router.ts#L15" + } + ], + "target": 2093 + }, + { + "id": 7719, + "name": "GlobInput", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 41, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L41" + } + ], + "target": 1211 + }, + { + "id": 7718, + "name": "GlobObject", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 36, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L36" + } + ], + "target": 1208 + }, + { + "id": 7787, + "name": "IRequestList", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_list.ts", + "line": 26, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_list.ts#L26" + } + ], + "target": 2992 + }, + { + "id": 7794, + "name": "IRequestManager", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 46, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L46" + } + ], + "target": 3161 + }, + { + "id": 7803, + "name": "IStorage", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/storage_manager.ts", + "line": 14, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/storage_manager.ts#L14" + } + ], + "target": 3328 + }, + { + "id": 7781, + "name": "KeyConsumer", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/key_value_store.ts", + "line": 727, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/key_value_store.ts#L727" + } + ], + "target": 2970 + }, + { + "id": 7780, + "name": "KeyValueStore", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/key_value_store.ts", + "line": 108, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/key_value_store.ts#L108" + } + ], + "target": 2889 + }, + { + "id": 7784, + "name": "KeyValueStoreIteratorOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/key_value_store.ts", + "line": 761, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/key_value_store.ts#L761" + } + ], + "target": 2986 + }, + { + "id": 7782, + "name": "KeyValueStoreOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/key_value_store.ts", + "line": 737, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/key_value_store.ts#L737" + } + ], + "target": 2977 + }, + { + "id": 7685, + "name": "LoadedRequest", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 19, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L19" + } + ], + "target": 695 + }, + { + "id": 7728, + "name": "LocalEventManager", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/events/local_event_manager.ts", + "line": 9, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/events/local_event_manager.ts#L9" + } + ], + "target": 1278 + }, + { + "id": 7729, + "name": "log", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 9, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1348 + }, + { + "id": 7730, + "name": "Log", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 14, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1349 + }, + { + "id": 7732, + "name": "Logger", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 29, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1419 + }, + { + "id": 7733, + "name": "LoggerJson", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 37, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1579 + }, + { + "id": 7735, + "name": "LoggerOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 5, + "character": 14, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L5" + } + ], + "target": 1906 + }, + { + "id": 7734, + "name": "LoggerText", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 49, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1740 + }, + { + "id": 7731, + "name": "LogLevel", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/log.ts", + "line": 4, + "character": 19, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/log.ts#L4" + } + ], + "target": 1411 + }, + { + "id": 7762, + "name": "MAX_POOL_SIZE", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/consts.ts", + "line": 3, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/consts.ts#L3" + } + ], + "target": 2509 + }, + { + "id": 7661, + "name": "NonRetryableError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 4, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L4" + } + ], + "target": 78 + }, + { + "id": 7761, + "name": "PERSIST_STATE_KEY", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/consts.ts", + "line": 2, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/consts.ts#L2" + } + ], + "target": 2508 + }, + { + "id": 7691, + "name": "PersistenceOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/statistics.ts", + "line": 41, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/statistics.ts#L41" + } + ], + "target": 900 + }, + { + "id": 7739, + "name": "ProxyConfiguration", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/proxy_configuration.ts", + "line": 135, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/proxy_configuration.ts#L135" + } + ], + "target": 1926 + }, + { + "id": 7736, + "name": "ProxyConfigurationFunction", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/proxy_configuration.ts", + "line": 8, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/proxy_configuration.ts#L8" + } + ], + "target": 1914 + }, + { + "id": 7737, + "name": "ProxyConfigurationOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/proxy_configuration.ts", + "line": 14, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/proxy_configuration.ts#L14" + } + ], + "target": 1919 + }, + { + "id": 7654, + "name": "PseudoUrl", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/index.ts", + "line": 17, + "character": 9, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/index.ts#L17" + } + ], + "target": 15 + }, + { + "id": 7717, + "name": "PseudoUrlInput", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 34, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L34" + } + ], + "target": 1207 + }, + { + "id": 7716, + "name": "PseudoUrlObject", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 29, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L29" + } + ], + "target": 1204 + }, + { + "id": 7806, + "name": "purgeDefaultStorages", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/utils.ts", + "line": 33, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/utils.ts#L33" + }, + { + "fileName": "packages/core/src/storages/utils.ts", + "line": 45, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/utils.ts#L45" + }, + { + "fileName": "packages/core/src/storages/utils.ts", + "line": 46, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/utils.ts#L46" + } + ], + "target": 3406 + }, + { + "id": 7742, + "name": "PushErrorMessageOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 561, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L561" + } + ], + "target": 1990 + }, + { + "id": 7660, + "name": "QueueOperationInfo", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/index.ts", + "line": 18, + "character": 68, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/index.ts#L18" + } + ], + "target": 74 + }, + { + "id": 7783, + "name": "RecordOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/key_value_store.ts", + "line": 744, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/key_value_store.ts#L744" + } + ], + "target": 2982 + }, + { + "id": 7830, + "name": "RecoverableState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/recoverable_state.ts", + "line": 75, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/recoverable_state.ts#L75" + } + ], + "target": 3663 + }, + { + "id": 7829, + "name": "RecoverableStateOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/recoverable_state.ts", + "line": 33, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/recoverable_state.ts#L33" + } + ], + "target": 3646 + }, + { + "id": 7828, + "name": "RecoverableStatePersistenceOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/recoverable_state.ts", + "line": 6, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/recoverable_state.ts#L6" + } + ], + "target": 3641 + }, + { + "id": 7721, + "name": "RegExpInput", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 48, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L48" + } + ], + "target": 1215 + }, + { + "id": 7720, + "name": "RegExpObject", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 43, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L43" + } + ], + "target": 1212 + }, + { + "id": 7745, + "name": "Request", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 585, + "character": 27, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L585" + } + ], + "target": 1999 + }, + { + "id": 7669, + "name": "RequestHandlerError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 57, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L57" + } + ], + "target": 225 + }, + { + "id": 7689, + "name": "RequestHandlerResult", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 172, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L172" + } + ], + "target": 814 + }, + { + "id": 7789, + "name": "RequestList", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_list.ts", + "line": 307, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_list.ts#L307" + } + ], + "target": 3024 + }, + { + "id": 7788, + "name": "RequestListOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_list.ts", + "line": 91, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_list.ts#L91" + } + ], + "target": 3014 + }, + { + "id": 7791, + "name": "RequestListSourcesFunction", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_list.ts", + "line": 1016, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_list.ts#L1016" + } + ], + "target": 3126 + }, + { + "id": 7790, + "name": "RequestListState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_list.ts", + "line": 1004, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_list.ts#L1004" + } + ], + "target": 3122 + }, + { + "id": 7819, + "name": "RequestManagerTandem", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_manager_tandem.ts", + "line": 22, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_manager_tandem.ts#L22" + } + ], + "target": 3532 + }, + { + "id": 7741, + "name": "RequestOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 448, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L448" + } + ], + "target": 1970 + }, + { + "id": 7795, + "name": "RequestProvider", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 104, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L104" + } + ], + "target": 3192 + }, + { + "id": 7796, + "name": "RequestProviderOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 918, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L918" + } + ], + "target": 3292 + }, + { + "id": 7764, + "name": "RequestQueue", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/index.ts", + "line": 7, + "character": 9, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/index.ts#L7" + } + ], + "target": 2614 + }, + { + "id": 7799, + "name": "RequestQueueOperationOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 945, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L945" + } + ], + "target": 3310 + }, + { + "id": 7797, + "name": "RequestQueueOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 934, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L934" + } + ], + "target": 3297 + }, + { + "id": 7763, + "name": "RequestQueueV1", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/index.ts", + "line": 6, + "character": 9, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/index.ts#L6" + } + ], + "target": 2510 + }, + { + "id": 7765, + "name": "RequestQueueV2", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/index.ts", + "line": 8, + "character": 25, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/index.ts#L8" + } + ], + "target": 2720 + }, + { + "id": 7793, + "name": "RequestsLike", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 41, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L41" + } + ], + "target": 3160 + }, + { + "id": 7740, + "name": "RequestState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 42, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L42" + } + ], + "target": 1961 + }, + { + "id": 7724, + "name": "RequestTransform", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 299, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L299" + } + ], + "target": 1224 + }, + { + "id": 7827, + "name": "ResponseLike", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/cookie_utils.ts", + "line": 7, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/cookie_utils.ts#L7" + } + ], + "target": 3634 + }, + { + "id": 7687, + "name": "RestrictedCrawlingContext", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 28, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L28" + } + ], + "target": 701 + }, + { + "id": 7664, + "name": "RetryRequestError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 22, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L22" + } + ], + "target": 138 + }, + { + "id": 7749, + "name": "Router", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/router.ts", + "line": 86, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/router.ts#L86" + } + ], + "target": 2103 + }, + { + "id": 7746, + "name": "RouterHandler", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/router.ts", + "line": 10, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/router.ts#L10" + } + ], + "target": 2058 + }, + { + "id": 7748, + "name": "RouterRoutes", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/router.ts", + "line": 17, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/router.ts#L17" + } + ], + "target": 2095 + }, + { + "id": 7756, + "name": "Session", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/session.ts", + "line": 84, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/session.ts#L84" + } + ], + "target": 2200 + }, + { + "id": 7665, + "name": "SessionError", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/errors.ts", + "line": 33, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/errors.ts#L33" + } + ], + "target": 155 + }, + { + "id": 7755, + "name": "SessionOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/session.ts", + "line": 20, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/session.ts#L20" + } + ], + "target": 2185 + }, + { + "id": 7759, + "name": "SessionPool", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/session_pool.ts", + "line": 137, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/session_pool.ts#L137" + } + ], + "target": 2293 + }, + { + "id": 7758, + "name": "SessionPoolOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/session_pool/session_pool.ts", + "line": 30, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/session_pool/session_pool.ts#L30" + } + ], + "target": 2284 + }, + { + "id": 7818, + "name": "SitemapRequestList", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/sitemap_request_list.ts", + "line": 133, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/sitemap_request_list.ts#L133" + } + ], + "target": 3456 + }, + { + "id": 7817, + "name": "SitemapRequestListOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/sitemap_request_list.ts", + "line": 61, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/sitemap_request_list.ts#L61" + } + ], + "target": 3440 + }, + { + "id": 7723, + "name": "SkippedRequestCallback", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 52, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L52" + } + ], + "target": 1217 + }, + { + "id": 7722, + "name": "SkippedRequestReason", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 50, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L50" + } + ], + "target": 1216 + }, + { + "id": 7699, + "name": "SnapshotResult", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/error_snapshotter.ts", + "line": 13, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/error_snapshotter.ts#L13" + } + ], + "target": 1068 + }, + { + "id": 7673, + "name": "Snapshotter", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/snapshotter.ts", + "line": 118, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/snapshotter.ts#L118" + } + ], + "target": 375 + }, + { + "id": 7672, + "name": "SnapshotterOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/snapshotter.ts", + "line": 19, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/snapshotter.ts#L19" + } + ], + "target": 365 + }, + { + "id": 7743, + "name": "Source", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 577, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L577" + } + ], + "target": 1992 + }, + { + "id": 7694, + "name": "StatisticPersistedState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/statistics.ts", + "line": 497, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/statistics.ts#L497" + } + ], + "target": 980 + }, + { + "id": 7692, + "name": "Statistics", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/statistics.ts", + "line": 59, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/statistics.ts#L59" + } + ], + "target": 902 + }, + { + "id": 7693, + "name": "StatisticsOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/statistics.ts", + "line": 441, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/statistics.ts#L441" + } + ], + "target": 971 + }, + { + "id": 7695, + "name": "StatisticState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/statistics.ts", + "line": 511, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/statistics.ts#L511" + } + ], + "target": 1004 + }, + { + "id": 7658, + "name": "StorageClient", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/index.ts", + "line": 18, + "character": 45, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/index.ts#L18" + } + ], + "target": 32 + }, + { + "id": 7805, + "name": "StorageManagerOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/storage_manager.ts", + "line": 158, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/storage_manager.ts#L158" + } + ], + "target": 3401 + }, + { + "id": 7674, + "name": "SystemInfo", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/system_status.ts", + "line": 10, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/system_status.ts#L10" + } + ], + "target": 443 + }, + { + "id": 7678, + "name": "SystemStatus", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/system_status.ts", + "line": 120, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/system_status.ts#L120" + } + ], + "target": 476 + }, + { + "id": 7675, + "name": "SystemStatusOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/autoscaling/system_status.ts", + "line": 35, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/autoscaling/system_status.ts#L35" + } + ], + "target": 453 + }, + { + "id": 7738, + "name": "TieredProxy", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/proxy_configuration.ts", + "line": 44, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/proxy_configuration.ts#L44" + } + ], + "target": 1923 + }, + { + "id": 7714, + "name": "tryAbsoluteURL", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 12, + "character": 9, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L12" + } + ], + "target": 1196 + }, + { + "id": 7715, + "name": "UrlPatternObject", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/shared.ts", + "line": 24, + "character": 12, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/shared.ts#L24" + } + ], + "target": 1200 + }, + { + "id": 7807, + "name": "useState", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/utils.ts", + "line": 87, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/utils.ts#L87" + } + ], + "target": 3412 + }, + { + "id": 7809, + "name": "UseStateOptions", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/utils.ts", + "line": 69, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/utils.ts#L69" + } + ], + "target": 3421 + }, + { + "id": 7816, + "name": "withCheckedStorageAccess", + "variant": "reference", + "kind": 4194304, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/storages/access_checking.ts", + "line": 18, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/access_checking.ts#L18" + } + ], + "target": 3431 + }, + { + "id": 7975, + "name": "BasicCrawler", + "variant": "declaration", + "kind": 128, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides a simple framework for parallel crawling of web pages.\nThe URLs to crawl are fed either from a static list of URLs\nor from a dynamic queue of URLs enabling recursive crawling of websites.\n\n" + }, + { + "kind": "code", + "text": "`BasicCrawler`" + }, + { + "kind": "text", + "text": " is a low-level tool that requires the user to implement the page\ndownload and data extraction functionality themselves.\nIf we want a crawler that already facilitates this functionality,\nwe should consider using " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "CheerioCrawler" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "PuppeteerCrawler" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "PlaywrightCrawler" + }, + { + "kind": "text", + "text": ".\n\n" + }, + { + "kind": "code", + "text": "`BasicCrawler`" + }, + { + "kind": "text", + "text": " invokes the user-provided " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestHandler|`requestHandler`" + }, + { + "kind": "text", + "text": "\nfor each " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " object, which represents a single URL to crawl.\nThe " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects are fed from the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestList" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": "\ninstances provided by the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestList|`requestList`" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestQueue|`requestQueue`" + }, + { + "kind": "text", + "text": "\nconstructor options, respectively. If neither " + }, + { + "kind": "code", + "text": "`requestList`" + }, + { + "kind": "text", + "text": " nor " + }, + { + "kind": "code", + "text": "`requestQueue`" + }, + { + "kind": "text", + "text": " options are provided,\nthe crawler will open the default request queue either when the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.addRequests|`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " function is called,\nor if " + }, + { + "kind": "code", + "text": "`requests`" + }, + { + "kind": "text", + "text": " parameter (representing the initial requests) of the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.run|`crawler.run()`" + }, + { + "kind": "text", + "text": " function is provided.\n\nIf both " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestList|`requestList`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestQueue|`requestQueue`" + }, + { + "kind": "text", + "text": " options are used,\nthe instance first processes URLs from the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestList" + }, + { + "kind": "text", + "text": " and automatically enqueues all of them\nto the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": " before it starts their processing. This ensures that a single URL is not crawled multiple times.\n\nThe crawler finishes if there are no more " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects to crawl.\n\nNew requests are only dispatched when there is enough free CPU and memory available,\nusing the functionality provided by the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool" + }, + { + "kind": "text", + "text": " class.\nAll " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool" + }, + { + "kind": "text", + "text": " configuration options can be passed to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`" + }, + { + "kind": "text", + "text": "\nparameter of the " + }, + { + "kind": "code", + "text": "`BasicCrawler`" + }, + { + "kind": "text", + "text": " constructor.\nFor user convenience, the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.minConcurrency|`minConcurrency`" + }, + { + "kind": "text", + "text": " and\n" + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`" + }, + { + "kind": "text", + "text": " options of the\nunderlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool" + }, + { + "kind": "text", + "text": " constructor are available directly in the " + }, + { + "kind": "code", + "text": "`BasicCrawler`" + }, + { + "kind": "text", + "text": " constructor.\n\n**Example usage:**\n\n" + }, + { + "kind": "code", + "text": "```javascript\nimport { BasicCrawler, Dataset } from 'crawlee';\n\n// Create a crawler instance\nconst crawler = new BasicCrawler({\n async requestHandler({ request, sendRequest }) {\n // 'request' contains an instance of the Request class\n // Here we simply fetch the HTML of the page and store it to a dataset\n const { body } = await sendRequest({\n url: request.url,\n method: request.method,\n body: request.payload,\n headers: request.headers,\n });\n\n await Dataset.pushData({\n url: request.url,\n html: body,\n })\n },\n});\n\n// Enqueue the initial requests and run the crawler\nawait crawler.run([\n 'http://www.example.com/page-1',\n 'http://www.example.com/page-2',\n]);\n```" + } + ] + }, + "children": [ + { + "id": 8011, + "name": "constructor", + "variant": "declaration", + "kind": 512, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 648, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L648" + } + ], + "signatures": [ + { + "id": 8012, + "name": "new BasicCrawler", + "variant": "signature", + "kind": 16384, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "All " + }, + { + "kind": "code", + "text": "`BasicCrawler`" + }, + { + "kind": "text", + "text": " parameters are passed via an options object." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 648, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L648" + } + ], + "typeParameters": [ + { + "id": 8013, + "name": "Context", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + "default": { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + } + }, + { + "id": 8014, + "name": "ContextExtension", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "never" + } + ], + "name": "Dictionary", + "package": "@crawlee/types" + } + }, + { + "id": 8015, + "name": "ExtendedContext", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + "default": { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8014, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ContextExtension", + "refersToTypeParameter": true + } + ] + } + } + ], + "parameters": [ + { + "id": 8016, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": 7930, + "typeArguments": [ + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8014, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ContextExtension", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8015, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "BasicCrawlerOptions", + "package": "@crawlee/basic" + }, + { + "type": "reference", + "target": 7922, + "typeArguments": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + } + ], + "name": "RequireContextPipeline", + "package": "@crawlee/basic" + } + ] + }, + "defaultValue": "..." + }, + { + "id": 8017, + "name": "config", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "reference", + "target": 556, + "name": "Configuration", + "package": "@crawlee/core" + }, + "defaultValue": "..." + } + ], + "type": { + "type": "reference", + "target": 7975, + "typeArguments": [ + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8014, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ContextExtension", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8015, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "BasicCrawler", + "package": "@crawlee/basic" + } + } + ] + }, + { + "id": 8023, + "name": "autoscaledPool", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool" + }, + { + "kind": "text", + "text": " class that manages the concurrency of the crawler.\n> *NOTE:* This property is only initialized after calling the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.run|`crawler.run()`" + }, + { + "kind": "text", + "text": " function.\nWe can use it to change the concurrency settings on the fly,\nto pause the crawler by calling " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool.pause|`autoscaledPool.pause()`" + }, + { + "kind": "text", + "text": "\nor to abort it by calling " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool.abort|`autoscaledPool.abort()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 539, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L539" + } + ], + "type": { + "type": "reference", + "target": 267, + "name": "AutoscaledPool", + "package": "@crawlee/core" + } + }, + { + "id": 8067, + "name": "config", + "variant": "declaration", + "kind": 1024, + "flags": { + "isReadonly": true + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 651, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L651" + } + ], + "type": { + "type": "reference", + "target": 556, + "name": "Configuration", + "package": "@crawlee/core" + }, + "defaultValue": "..." + }, + { + "id": 8033, + "name": "hasFinishedBefore", + "variant": "declaration", + "kind": 1024, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 565, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L565" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + }, + "defaultValue": "false" + }, + { + "id": 8034, + "name": "log", + "variant": "declaration", + "kind": 1024, + "flags": { + "isReadonly": true + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 567, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L567" + } + ], + "type": { + "type": "reference", + "target": 1349, + "name": "Log", + "package": "@apify/log" + } + }, + { + "id": 8024, + "name": "proxyConfiguration", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "ProxyConfiguration" + }, + { + "kind": "text", + "text": " class that manages the crawler's proxies.\nOnly available if used by the crawler." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 545, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L545" + } + ], + "type": { + "type": "reference", + "target": 1926, + "name": "ProxyConfiguration", + "package": "@crawlee/core" + } + }, + { + "id": 8019, + "name": "requestList", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestList" + }, + { + "kind": "text", + "text": " class that manages the crawler's " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request|requests" + }, + { + "kind": "text", + "text": ".\nOnly available if used by the crawler." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 512, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L512" + } + ], + "type": { + "type": "reference", + "target": 2992, + "name": "IRequestList", + "package": "@crawlee/core" + } + }, + { + "id": 8020, + "name": "requestQueue", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.\nA reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": " class that manages the crawler's " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request|requests" + }, + { + "kind": "text", + "text": ".\nOnly available if used by the crawler." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 519, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L519" + } + ], + "type": { + "type": "reference", + "target": 3192, + "name": "RequestProvider", + "package": "@crawlee/core" + } + }, + { + "id": 8025, + "name": "router", + "variant": "declaration", + "kind": 1024, + "flags": { + "isReadonly": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Router" + }, + { + "kind": "text", + "text": " instance that will be used if we don't specify any " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestHandler|`requestHandler`" + }, + { + "kind": "text", + "text": ".\nSee " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Router.addHandler|`router.addHandler()`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Router.addDefaultHandler|`router.addDefaultHandler()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 551, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L551" + } + ], + "type": { + "type": "reference", + "target": 2058, + "typeArguments": [ + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + } + ], + "name": "RouterHandler", + "package": "@crawlee/core" + }, + "defaultValue": "..." + }, + { + "id": 8032, + "name": "running", + "variant": "declaration", + "kind": 1024, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 564, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L564" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + }, + "defaultValue": "false" + }, + { + "id": 8022, + "name": "sessionPool", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "SessionPool" + }, + { + "kind": "text", + "text": " class that manages the crawler's " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Session|sessions" + }, + { + "kind": "text", + "text": ".\nOnly available if used by the crawler." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 530, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L530" + } + ], + "type": { + "type": "reference", + "target": 2293, + "name": "SessionPool", + "package": "@crawlee/core" + } + }, + { + "id": 8018, + "name": "stats", + "variant": "declaration", + "kind": 1024, + "flags": { + "isReadonly": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reference to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Statistics" + }, + { + "kind": "text", + "text": " class that collects and logs run statistics for requests." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 506, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L506" + } + ], + "type": { + "type": "reference", + "target": 902, + "name": "Statistics", + "package": "@crawlee/core" + } + }, + { + "id": 8030, + "name": "contextPipeline", + "variant": "declaration", + "kind": 262144, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 556, + "character": 8, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L556" + } + ], + "getSignature": { + "id": 8031, + "name": "contextPipeline", + "variant": "signature", + "kind": 524288, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 556, + "character": 8, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L556" + } + ], + "type": { + "type": "reference", + "target": 669, + "typeArguments": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 8015, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "ContextPipeline", + "package": "@crawlee/core" + } + } + }, + { + "id": 8108, + "name": "addRequests", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1204, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1204" + } + ], + "signatures": [ + { + "id": 8109, + "name": "addRequests", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue\nadding the rest in background. You can configure the batch size via " + }, + { + "kind": "code", + "text": "`batchSize`" + }, + { + "kind": "text", + "text": " option and the sleep time in between\nthe batches via " + }, + { + "kind": "code", + "text": "`waitBetweenBatchesMillis`" + }, + { + "kind": "text", + "text": ". If you want to wait for all batches to be added to the queue, you can use\nthe " + }, + { + "kind": "code", + "text": "`waitForAllRequestsToBeAdded`" + }, + { + "kind": "text", + "text": " promise you get in the response object.\n\nThis is an alias for calling " + }, + { + "kind": "code", + "text": "`addRequestsBatched()`" + }, + { + "kind": "text", + "text": " on the implicit " + }, + { + "kind": "code", + "text": "`RequestQueue`" + }, + { + "kind": "text", + "text": " for this crawler instance." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1204, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1204" + } + ], + "parameters": [ + { + "id": 8110, + "name": "requests", + "variant": "param", + "kind": 32768, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The requests to add" + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyDeep" + }, + "typeArguments": [ + { + "type": "reference", + "target": 3160, + "name": "RequestsLike", + "package": "@crawlee/core" + } + ], + "name": "ReadonlyDeep", + "package": "type-fest" + } + }, + { + "id": 8111, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Options for the request queue" + } + ] + }, + "type": { + "type": "reference", + "target": 8254, + "name": "CrawlerAddRequestsOptions", + "package": "@crawlee/basic" + }, + "defaultValue": "{}" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 8260, + "name": "CrawlerAddRequestsResult", + "package": "@crawlee/basic" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8122, + "name": "exportData", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1310, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1310" + } + ], + "signatures": [ + { + "id": 8123, + "name": "exportData", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieves all the data from the default crawler " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": " and exports them to the specified format.\nSupported formats are currently 'json' and 'csv', and will be inferred from the " + }, + { + "kind": "code", + "text": "`path`" + }, + { + "kind": "text", + "text": " automatically." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1310, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1310" + } + ], + "typeParameters": [ + { + "id": 8124, + "name": "Data", + "variant": "typeParam", + "kind": 131072, + "flags": {} + } + ], + "parameters": [ + { + "id": 8125, + "name": "path", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 8126, + "name": "format", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": "json" + }, + { + "type": "literal", + "value": "csv" + } + ] + } + }, + { + "id": 8127, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "reference", + "target": 2741, + "name": "DatasetExportOptions", + "package": "@crawlee/core" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": 8124, + "name": "Data", + "package": "@crawlee/basic", + "refersToTypeParameter": true + } + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8119, + "name": "getData", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1301, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1301" + } + ], + "signatures": [ + { + "id": 8120, + "name": "getData", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieves data from the default crawler " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": " by calling " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset.getData" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1301, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1301" + } + ], + "parameters": [ + { + "id": 8121, + "name": "args", + "variant": "param", + "kind": 32768, + "flags": { + "isRest": true + }, + "type": { + "type": "tuple", + "elements": [ + { + "type": "namedTupleMember", + "name": "options", + "isOptional": false, + "element": { + "type": "reference", + "target": 2732, + "name": "DatasetDataOptions", + "package": "@crawlee/core" + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 2874, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "DatasetContent", + "package": "@crawlee/core" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8116, + "name": "getDataset", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1294, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1294" + } + ], + "signatures": [ + { + "id": 8117, + "name": "getDataset", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieves the specified " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": ", or the default crawler " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1294, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1294" + } + ], + "parameters": [ + { + "id": 8118, + "name": "idOrName", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 2769, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "Dataset", + "package": "@crawlee/core" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8091, + "name": "getRequestQueue", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1113, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1113" + } + ], + "signatures": [ + { + "id": 8092, + "name": "getRequestQueue", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1113, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1113" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 3192, + "name": "RequestProvider", + "package": "@crawlee/core" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8112, + "name": "pushData", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1286, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1286" + } + ], + "signatures": [ + { + "id": 8113, + "name": "pushData", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes data to the specified " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": ", or the default crawler " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": " by calling " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset.pushData" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1286, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1286" + } + ], + "parameters": [ + { + "id": 8114, + "name": "data", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + } + ] + } + }, + { + "id": 8115, + "name": "datasetIdOrName", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "void" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8084, + "name": "run", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 972, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L972" + } + ], + "signatures": [ + { + "id": 8085, + "name": "run", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Runs the crawler. Returns a promise that resolves once all the requests are processed\nand " + }, + { + "kind": "code", + "text": "`autoscaledPool.isFinished`" + }, + { + "kind": "text", + "text": " returns " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": ".\n\nWe can use the " + }, + { + "kind": "code", + "text": "`requests`" + }, + { + "kind": "text", + "text": " parameter to enqueue the initial requests — it is a shortcut for\nrunning " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.addRequests|`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " before " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.run|`crawler.run()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 972, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L972" + } + ], + "parameters": [ + { + "id": 8086, + "name": "requests", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The requests to add." + } + ] + }, + "type": { + "type": "reference", + "target": 3160, + "name": "RequestsLike", + "package": "@crawlee/core" + } + }, + { + "id": 8087, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Options for the request queue." + } + ] + }, + "type": { + "type": "reference", + "target": 8263, + "name": "CrawlerRunOptions", + "package": "@crawlee/basic" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 465, + "name": "FinalStatistics", + "package": "@crawlee/core" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8071, + "name": "setStatusMessage", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 895, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L895" + } + ], + "signatures": [ + { + "id": 8072, + "name": "setStatusMessage", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "This method is periodically called by the crawler, every " + }, + { + "kind": "code", + "text": "`statusMessageLoggingInterval`" + }, + { + "kind": "text", + "text": " seconds." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 895, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L895" + } + ], + "parameters": [ + { + "id": 8073, + "name": "message", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 8074, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "reference", + "target": 15806, + "name": "SetStatusMessageOptions", + "package": "@crawlee/types" + }, + "defaultValue": "{}" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "void" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8088, + "name": "stop", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1101, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1101" + } + ], + "signatures": [ + { + "id": 8089, + "name": "stop", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gracefully stops the current run of the crawler.\n\nAll the tasks active at the time of calling this method will be allowed to finish.\n\nTo stop the crawler immediately, use " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.teardown|`crawler.teardown()`" + }, + { + "kind": "text", + "text": " instead." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1101, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1101" + } + ], + "parameters": [ + { + "id": 8090, + "name": "message", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "intrinsic", + "name": "string" + }, + "defaultValue": "'The crawler has been gracefully stopped.'" + } + ], + "type": { + "type": "intrinsic", + "name": "void" + } + } + ] + }, + { + "id": 8237, + "name": "teardown", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1992, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1992" + } + ], + "signatures": [ + { + "id": 8238, + "name": "teardown", + "variant": "signature", + "kind": 4096, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Stops the crawler immediately.\n\nThis method doesn't wait for currently active requests to finish.\n\nTo stop the crawler gracefully (waiting for all running requests to finish), use " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.stop|`crawler.stop()`" + }, + { + "kind": "text", + "text": " instead." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1992, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1992" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "void" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + }, + { + "id": 8093, + "name": "useState", + "variant": "declaration", + "kind": 2048, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1135, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1135" + } + ], + "signatures": [ + { + "id": 8094, + "name": "useState", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 1135, + "character": 10, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L1135" + } + ], + "typeParameters": [ + { + "id": 8095, + "name": "State", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + }, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + } + ], + "parameters": [ + { + "id": 8096, + "name": "defaultValue", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "reference", + "target": 8095, + "name": "State", + "package": "@crawlee/basic", + "refersToTypeParameter": true + }, + "defaultValue": "..." + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 8095, + "name": "State", + "package": "@crawlee/basic", + "refersToTypeParameter": true + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 8011 + ] + }, + { + "title": "Properties", + "children": [ + 8023, + 8067, + 8033, + 8034, + 8024, + 8019, + 8020, + 8025, + 8032, + 8022, + 8018 + ] + }, + { + "title": "Accessors", + "children": [ + 8030 + ] + }, + { + "title": "Methods", + "children": [ + 8108, + 8122, + 8119, + 8116, + 8091, + 8112, + 8084, + 8071, + 8088, + 8237, + 8093 + ] + } + ], + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 490, + "character": 13, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L490" + } + ], + "typeParameters": [ + { + "id": 8247, + "name": "Context", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 753, + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + "default": { + "type": "reference", + "target": 753, + "name": "CrawlingContext", + "package": "@crawlee/core" + } + }, + { + "id": 8248, + "name": "ContextExtension", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "never" + } + ], + "name": "Dictionary", + "package": "@crawlee/types" + } + }, + { + "id": 8249, + "name": "ExtendedContext", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + }, + "default": { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": 8013, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.Context", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 8014, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawler.ContextExtension", + "refersToTypeParameter": true + } + ] + } + } + ], + "extendedBy": [ + { + "type": "reference", + "target": 8589, + "name": "BrowserCrawler" + }, + { + "type": "reference", + "target": 9398, + "name": "HttpCrawler" + }, + { + "type": "reference", + "target": 9832, + "name": "FileDownload" + }, + { + "type": "reference", + "target": 13401, + "name": "AdaptivePlaywrightCrawler" + } + ] + }, + { + "id": 7039, + "name": "Cheerio", + "variant": "declaration", + "kind": 128, + "flags": { + "isExternal": true, + "isAbstract": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The cheerio class is the central class of the library. It wraps a set of\nelements and provides an API for traversing, modifying, and interacting with\nthe set.\n\nLoading a document will return the Cheerio class bound to the root element of\nthe document. The class will be instantiated when querying the document (when\ncalling " + }, + { + "kind": "code", + "text": "`$('selector')`" + }, + { + "kind": "text", + "text": ")." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "This is the HTML markup we will be using in all of the API examples:", + "content": [ + { + "kind": "code", + "text": "```html\n
    \n
  • Apple
    • \n
  • Orange
    • \n
  • Pear
    • \n
\n```" + } + ] + } + ] + }, + "children": [ + { + "id": 7064, + "name": "cheerio", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 81, + "character": 4 + } + ], + "type": { + "type": "literal", + "value": "[cheerio object]" + } + }, + { + "id": 7046, + "name": "length", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 31, + "character": 4 + } + ], + "type": { + "type": "intrinsic", + "name": "number" + }, + "implementationOf": { + "type": "reference", + "target": -1, + "name": "ArrayLike.length" + } + }, + { + "id": 7047, + "name": "options", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 33, + "character": 4 + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/options.ts", + "qualifiedName": "InternalOptions" + }, + "name": "InternalOptions", + "package": "cheerio" + } + }, + { + "id": 7049, + "name": "prevObject", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 51, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "intrinsic", + "name": "any" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + }, + { + "id": 7065, + "name": "splice", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 82, + "character": 4 + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7066, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "website/node_modules/typescript/lib/lib.es5.d.ts", + "line": 1404, + "character": 4 + }, + { + "fileName": "website/node_modules/typescript/lib/lib.es5.d.ts", + "line": 1414, + "character": 4 + } + ], + "signatures": [ + { + "id": 7067, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Removes elements from an array and, if necessary, inserts new elements in their place, returning the deleted elements." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "An array containing the elements that were deleted." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "website/node_modules/typescript/lib/lib.es5.d.ts", + "line": 1404, + "character": 4 + } + ], + "parameters": [ + { + "id": 7068, + "name": "start", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The zero-based location in the array from which to start removing elements." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7069, + "name": "deleteCount", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of elements to remove. Omitting this argument will remove all elements from the start\nparamater location to end of the array. If value of this argument is either a negative number, zero, undefined, or a type\nthat cannot be converted to an integer, the function will evaluate the argument as zero and not remove any elements." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "any" + } + } + }, + { + "id": 7070, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Removes elements from an array and, if necessary, inserts new elements in their place, returning the deleted elements." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "An array containing the elements that were deleted." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "website/node_modules/typescript/lib/lib.es5.d.ts", + "line": 1414, + "character": 4 + } + ], + "parameters": [ + { + "id": 7071, + "name": "start", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The zero-based location in the array from which to start removing elements." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7072, + "name": "deleteCount", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of elements to remove. If value of this argument is either a negative number, zero,\nundefined, or a type that cannot be converted to an integer, the function will evaluate the argument as zero and\nnot remove any elements." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7073, + "name": "items", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isRest": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Elements to insert into the array in place of the deleted elements." + } + ] + }, + "type": { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "any" + } + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "any" + } + } + } + ] + } + } + }, + { + "id": 7590, + "name": "[iterator]", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "website/node_modules/typescript/lib/lib.es2015.iterable.d.ts", + "line": 49, + "character": 4 + } + ], + "signatures": [ + { + "id": 7591, + "name": "[iterator]", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "website/node_modules/typescript/lib/lib.es2015.iterable.d.ts", + "line": 49, + "character": 4 + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es2015.iterable.d.ts", + "qualifiedName": "Iterator" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7592, + "name": "T", + "package": "cheerio", + "qualifiedName": "Cheerio.T", + "refersToTypeParameter": true + }, + { + "type": "intrinsic", + "name": "any" + }, + { + "type": "intrinsic", + "name": "any" + } + ], + "name": "Iterator", + "package": "typescript" + } + } + ] + }, + { + "id": 7351, + "name": "add", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 639, + "character": 24 + } + ], + "signatures": [ + { + "id": 7352, + "name": "add", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add elements to the set of matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').add('.orange').length;\n//=> 2\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The combined set." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/add/", + "target": "https://api.jquery.com/add/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 639, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7353, + "name": "S", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7354, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7355, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7354, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7356, + "name": "other", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Elements to add." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": 7353, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7353, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": 7353, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + } + }, + { + "id": 7357, + "name": "context", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optionally the context of the new selection." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7353, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "target": 7353, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 7354, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ] + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7358, + "name": "addBack", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 656, + "character": 24 + } + ], + "signatures": [ + { + "id": 7359, + "name": "addBack", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add the previous set of elements on the stack to the current set, optionally\nfiltered by a selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').eq(0).addBack('.orange').length;\n//=> 2\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The combined set." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/addBack/", + "target": "https://api.jquery.com/addBack/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 656, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7360, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7361, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7360, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7362, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for the elements to add." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7198, + "name": "addClass", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 340, + "character": 24 + } + ], + "signatures": [ + { + "id": 7199, + "name": "addClass", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Adds class(es) to all of the matched elements. Also accepts a " + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').addClass('fruit').html();\n//=>
  • Pear
    • \n\n$('.apple').addClass('fruit red').html();\n//=>
  • Apple
    • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/addClass/", + "target": "https://api.jquery.com/addClass/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 340, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7200, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7201, + "name": "R", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7200, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + } + ], + "parameters": [ + { + "id": 7202, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7201, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7203, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of new class." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7204, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 340, + "character": 103 + } + ], + "signatures": [ + { + "id": 7205, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 340, + "character": 103 + } + ], + "parameters": [ + { + "id": 7206, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7207, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7208, + "name": "className", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "intrinsic", + "name": "string" + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7201, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + }, + { + "id": 7447, + "name": "after", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 307, + "character": 24 + } + ], + "signatures": [ + { + "id": 7448, + "name": "after", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert content next to each element in the set of matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').after('
  • Plum
    • ');\n$.html();\n//=>
      \n//
    • Apple
      • \n//
    • Plum
      • \n//
    • Orange
      • \n//
    • Pear
      • \n//
    \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/after/", + "target": "https://api.jquery.com/after/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 307, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7449, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7450, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7449, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7451, + "name": "elems", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isRest": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTML string, DOM element, array of DOM elements or Cheerio to\n insert after each element in the set of matched elements." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + }, + { + "type": "tuple", + "elements": [ + { + "type": "reflection", + "declaration": { + "id": 7452, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 307, + "character": 78 + } + ], + "signatures": [ + { + "id": 7453, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 307, + "character": 78 + } + ], + "parameters": [ + { + "id": 7454, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7455, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7456, + "name": "html", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ] + } + } + ] + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7449, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7520, + "name": "append", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 84, + "character": 21 + } + ], + "signatures": [ + { + "id": 7521, + "name": "append", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Inserts content as the _last_ child of each of the selected elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').append('
  • Plum
    • ');\n$.html();\n//=>
      \n//
    • Apple
      • \n//
    • Orange
      • \n//
    • Pear
      • \n//
    • Plum
      • \n//
    \n```" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/append/", + "target": "https://api.jquery.com/append/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 84, + "character": 29 + } + ], + "typeParameters": [ + { + "id": 7522, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7523, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7522, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7524, + "name": "elems", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isRest": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + }, + { + "type": "tuple", + "elements": [ + { + "type": "reflection", + "declaration": { + "id": 7525, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 84, + "character": 78 + } + ], + "signatures": [ + { + "id": 7526, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 84, + "character": 78 + } + ], + "parameters": [ + { + "id": 7527, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7528, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7529, + "name": "html", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ] + } + } + ] + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7522, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7427, + "name": "appendTo", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 41, + "character": 24 + } + ], + "signatures": [ + { + "id": 7428, + "name": "appendTo", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert every element in the set of matched elements to the end of the target." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
  • Plum
    • ').appendTo('#fruits');\n$.html();\n//=>
      \n//
    • Apple
      • \n//
    • Orange
      • \n//
    • Pear
      • \n//
    • Plum
      • \n//
    \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/appendTo/", + "target": "https://api.jquery.com/appendTo/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 41, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7429, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7430, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7429, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7431, + "name": "target", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to append elements to." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7429, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7074, + "name": "attr", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 24, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 40, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 59, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 77, + "character": 24 + } + ], + "signatures": [ + { + "id": 7075, + "name": "attr", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting attributes. Gets the attribute value for only the first\nelement in the matched set." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').attr('id');\n//=> fruits\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The attribute's value." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/attr/", + "target": "https://api.jquery.com/attr/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 24, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7076, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7077, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7076, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7078, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the attribute." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7079, + "name": "attr", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting all attributes and their values of the first element in\nthe matched set." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').attr();\n//=> { id: 'fruits' }\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The attribute's values." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/attr/", + "target": "https://api.jquery.com/attr/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 40, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7080, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7081, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7080, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7082, + "name": "attr", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for setting attributes. Sets the attribute value for only the first\nelement in the matched set. If you set an attribute's value to " + }, + { + "kind": "code", + "text": "`null`" + }, + { + "kind": "text", + "text": ", you\nremove that attribute. You may also pass a " + }, + { + "kind": "code", + "text": "`map`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').attr('id', 'favorite').html();\n//=>
  • Apple
    • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/attr/", + "target": "https://api.jquery.com/attr/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 59, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7083, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7084, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7083, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7085, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the attribute." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7086, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new value of the attribute." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7087, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 59, + "character": 105 + } + ], + "signatures": [ + { + "id": 7088, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 59, + "character": 105 + } + ], + "parameters": [ + { + "id": 7089, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7090, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7091, + "name": "attrib", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7083, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7092, + "name": "attr", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for setting multiple attributes at once. Sets the attribute value for\nonly the first element in the matched set. If you set an attribute's value to\n" + }, + { + "kind": "code", + "text": "`null`" + }, + { + "kind": "text", + "text": ", you remove that attribute." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').attr({ id: 'favorite' }).html();\n//=>
  • Apple
    • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/attr/", + "target": "https://api.jquery.com/attr/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 77, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7093, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7094, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7093, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7095, + "name": "values", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Map of attribute names and values." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + } + ] + } + ], + "name": "Record", + "package": "typescript" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7093, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7462, + "name": "before", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 352, + "character": 24 + } + ], + "signatures": [ + { + "id": 7463, + "name": "before", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert content previous to each element in the set of matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').before('
  • Plum
    • ');\n$.html();\n//=>
      \n//
    • Plum
      • \n//
    • Apple
      • \n//
    • Orange
      • \n//
    • Pear
      • \n//
    \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/before/", + "target": "https://api.jquery.com/before/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 352, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7464, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7465, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7464, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7466, + "name": "elems", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isRest": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTML string, DOM element, array of DOM elements or Cheerio to\n insert before each element in the set of matched elements." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + }, + { + "type": "tuple", + "elements": [ + { + "type": "reflection", + "declaration": { + "id": 7467, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 352, + "character": 79 + } + ], + "signatures": [ + { + "id": 7468, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 352, + "character": 79 + } + ], + "parameters": [ + { + "id": 7469, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7470, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7471, + "name": "html", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ] + } + } + ] + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7464, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7416, + "name": "children", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 267, + "character": 21 + } + ], + "signatures": [ + { + "id": 7417, + "name": "children", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets the element children of each element in the set of matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('#fruits').children().length;\n//=> 3\n\n$('#fruits').children('.pear').text();\n//=> Pear\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The children." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/children/", + "target": "https://api.jquery.com/children/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 267, + "character": 31 + } + ], + "typeParameters": [ + { + "id": 7418, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7419, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7418, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7420, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for children." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7516, + "name": "clone", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 527, + "character": 24 + } + ], + "signatures": [ + { + "id": 7517, + "name": "clone", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone the cheerio object." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst moreFruit = $('#fruits').clone();\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The cloned object." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/clone/", + "target": "https://api.jquery.com/clone/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 527, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7518, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7519, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7518, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7518, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7244, + "name": "closest", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 119, + "character": 24 + } + ], + "signatures": [ + { + "id": 7245, + "name": "closest", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "For each element in the set, get the first element that matches the selector\nby testing the element itself and traversing up through its ancestors in the\nDOM tree." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').closest();\n//=> []\n\n$('.orange').closest('.apple');\n// => []\n\n$('.orange').closest('li');\n//=> [
  • Orange
    • ]\n\n$('.orange').closest('#fruits');\n//=> [
      ...
    ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The closest nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/closest/", + "target": "https://api.jquery.com/closest/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 119, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7246, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7247, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7246, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7248, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for the element to find." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7249, + "name": "contents", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 283, + "character": 24 + } + ], + "signatures": [ + { + "id": 7250, + "name": "contents", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets the children of each element in the set of matched elements, including\ntext and comment nodes." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('#fruits').contents().length;\n//=> 3\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The children." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/contents/", + "target": "https://api.jquery.com/contents/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 283, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7251, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7252, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7251, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7550, + "name": "css", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 12, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 22, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 32, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 41, + "character": 24 + } + ], + "signatures": [ + { + "id": 7551, + "name": "css", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a style property for the first element in the set of matched\nelements." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "A map of all of the style properties." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/css/", + "target": "https://api.jquery.com/css/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 12, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7552, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7553, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7552, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7554, + "name": "names", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optionally the names of the properties of interest." + } + ] + }, + "type": { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "string" + } + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7555, + "name": "css", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a style property for the first element in the set of matched\nelements." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The property value for the given name." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/css/", + "target": "https://api.jquery.com/css/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 22, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7556, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7557, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7556, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7558, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the property." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7559, + "name": "css", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set one CSS property for every matched element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/css/", + "target": "https://api.jquery.com/css/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 32, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7560, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7561, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7560, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7562, + "name": "prop", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the property." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7563, + "name": "val", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new value." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7564, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 32, + "character": 94 + } + ], + "signatures": [ + { + "id": 7565, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 32, + "character": 94 + } + ], + "parameters": [ + { + "id": 7566, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7567, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7568, + "name": "style", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "intrinsic", + "name": "string" + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7560, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7569, + "name": "css", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set multiple CSS properties for every matched element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/css/", + "target": "https://api.jquery.com/css/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/css.d.ts", + "line": 41, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7570, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7571, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7570, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7572, + "name": "map", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A map of property names and values." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7570, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7163, + "name": "data", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 191, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 207, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 227, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 246, + "character": 24 + } + ], + "signatures": [ + { + "id": 7164, + "name": "data", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting data attributes, for only the first element in the matched\nset." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
    ').data('apple-color');\n//=> 'red'\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The data attribute's value, or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": " if the attribute does not\n exist." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/data/", + "target": "https://api.jquery.com/data/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 191, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7165, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7166, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7165, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7167, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the data attribute." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "unknown" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7168, + "name": "data", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting all of an element's data attributes, for only the first\nelement in the matched set." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
    ').data();\n//=> { appleColor: 'red' }\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "A map with all of the data attributes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/data/", + "target": "https://api.jquery.com/data/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 207, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7169, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7170, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7169, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "unknown" + } + ], + "name": "Record", + "package": "typescript" + } + }, + { + "id": 7171, + "name": "data", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for setting data attributes, for only the first element in the matched\nset." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst apple = $('.apple').data('kind', 'mac');\n\napple.data('kind');\n//=> 'mac'\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/data/", + "target": "https://api.jquery.com/data/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 227, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7172, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7173, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7172, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7174, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the data attribute." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7175, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new value." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "unknown" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7172, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7176, + "name": "data", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for setting multiple data attributes at once, for only the first\nelement in the matched set." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst apple = $('.apple').data({ kind: 'mac' });\n\napple.data('kind');\n//=> 'mac'\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/data/", + "target": "https://api.jquery.com/data/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 246, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7177, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7178, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7177, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7179, + "name": "values", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Map of names to values." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "unknown" + } + ], + "name": "Record", + "package": "typescript" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7177, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7253, + "name": "each", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 309, + "character": 24 + } + ], + "signatures": [ + { + "id": 7254, + "name": "each", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterates over a cheerio object, executing a function for each matched\nelement. When the callback is fired, the function is fired in the context of\nthe DOM element, so " + }, + { + "kind": "code", + "text": "`this`" + }, + { + "kind": "text", + "text": " refers to the current element, which is equivalent\nto the function parameter " + }, + { + "kind": "code", + "text": "`element`" + }, + { + "kind": "text", + "text": ". To break out of the " + }, + { + "kind": "code", + "text": "`each`" + }, + { + "kind": "text", + "text": " loop early,\nreturn with " + }, + { + "kind": "code", + "text": "`false`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst fruits = [];\n\n$('li').each(function (i, elem) {\n fruits[i] = $(this).text();\n});\n\nfruits.join(', ');\n//=> Apple, Orange, Pear\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself, useful for chaining." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/each/", + "target": "https://api.jquery.com/each/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 309, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7255, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7256, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7255, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7257, + "name": "fn", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function to execute." + } + ] + }, + "type": { + "type": "reflection", + "declaration": { + "id": 7258, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 309, + "character": 54 + } + ], + "signatures": [ + { + "id": 7259, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 309, + "character": 54 + } + ], + "parameters": [ + { + "id": 7260, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7255, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7261, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7262, + "name": "el", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7255, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "boolean" + }, + { + "type": "intrinsic", + "name": "void" + } + ] + } + } + ] + } + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7255, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7487, + "name": "empty", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 434, + "character": 24 + } + ], + "signatures": [ + { + "id": 7488, + "name": "empty", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Removes all children from each item in the selection. Text nodes and comment\nnodes are left as is." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').empty();\n$.html();\n//=>
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/empty/", + "target": "https://api.jquery.com/empty/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 434, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7489, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7490, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7489, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7489, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7347, + "name": "end", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 622, + "character": 24 + } + ], + "signatures": [ + { + "id": 7348, + "name": "end", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "End the most recent filtering operation in the current chain and return the\nset of matched elements to its previous state." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').eq(0).end().length;\n//=> 3\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The previous state of the set of matched elements." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/end/", + "target": "https://api.jquery.com/end/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 622, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7349, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7350, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7349, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7319, + "name": "eq", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 519, + "character": 24 + } + ], + "signatures": [ + { + "id": 7320, + "name": "eq", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reduce the set of matched elements to the one at the specified index. Use\n" + }, + { + "kind": "code", + "text": "`.eq(-i)`" + }, + { + "kind": "text", + "text": " to count backwards from the last selected element." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').eq(0).text();\n//=> Apple\n\n$('li').eq(-1).text();\n//=> Pear\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The element at the " + }, + { + "kind": "code", + "text": "`i`" + }, + { + "kind": "text", + "text": "th position." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/eq/", + "target": "https://api.jquery.com/eq/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 519, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7321, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7322, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7321, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7323, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Index of the element to select." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7321, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7584, + "name": "extract", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/extract.d.ts", + "line": 27, + "character": 24 + } + ], + "signatures": [ + { + "id": 7585, + "name": "extract", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extract multiple values from a document, and store them in an object." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "An object containing the extracted values." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/extract.d.ts", + "line": 27, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7586, + "name": "M", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/api/extract.ts", + "qualifiedName": "ExtractMap" + }, + "name": "ExtractMap", + "package": "cheerio" + } + }, + { + "id": 7587, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7588, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7587, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7589, + "name": "map", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An object containing key-value pairs. The keys are the names of\n the properties to be created on the object, and the values are the\n selectors to be used to extract the values." + } + ] + }, + "type": { + "type": "reference", + "target": 7586, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/api/extract.ts", + "qualifiedName": "ExtractedMap" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7586, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ExtractedMap", + "package": "cheerio" + } + } + ] + }, + { + "id": 7274, + "name": "filter", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 361, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 397, + "character": 24 + } + ], + "signatures": [ + { + "id": 7275, + "name": "filter", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterates over a cheerio object, reducing the set of selector elements to\nthose that match the selector or pass the function's test.\n\nThis is the definition for using type guards; have a look below for other\nways to invoke this method. The function is executed in the context of the\nselected element, so " + }, + { + "kind": "code", + "text": "`this`" + }, + { + "kind": "text", + "text": " refers to the current element." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "Function", + "content": [ + { + "kind": "code", + "text": "```js\n$('li')\n .filter(function (i, el) {\n // this === el\n return $(this).attr('class') === 'orange';\n })\n .attr('class'); //=> orange\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The filtered collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/filter/", + "target": "https://api.jquery.com/filter/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 361, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7276, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + }, + { + "id": 7277, + "name": "S", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7278, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7276, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7279, + "name": "match", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to look for, following the rules above." + } + ] + }, + "type": { + "type": "reflection", + "declaration": { + "id": 7280, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 361, + "character": 72 + } + ], + "signatures": [ + { + "id": 7281, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 361, + "character": 72 + } + ], + "parameters": [ + { + "id": 7282, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7276, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7283, + "name": "index", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7284, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7276, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "predicate", + "name": "value", + "asserts": false, + "targetType": { + "type": "reference", + "target": 7277, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + } + } + ] + } + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7277, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7285, + "name": "filter", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterates over a cheerio object, reducing the set of selector elements to\nthose that match the selector or pass the function's test.\n\n- When a Cheerio selection is specified, return only the elements contained in\n that selection.\n- When an element is specified, return only that element (if it is contained in\n the original selection).\n- If using the function method, the function is executed in the context of the\n selected element, so " + }, + { + "kind": "code", + "text": "`this`" + }, + { + "kind": "text", + "text": " refers to the current element." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "Selector", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').filter('.orange').attr('class');\n//=> orange\n```" + } + ] + }, + { + "tag": "@example", + "name": "Function", + "content": [ + { + "kind": "code", + "text": "```js\n$('li')\n .filter(function (i, el) {\n // this === el\n return $(this).attr('class') === 'orange';\n })\n .attr('class'); //=> orange\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The filtered collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/filter/", + "target": "https://api.jquery.com/filter/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 397, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7286, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + }, + { + "id": 7287, + "name": "S", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7288, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7286, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7289, + "name": "match", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to look for, following the rules above. See\n " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "AcceptedFilters", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + } + }, + { + "kind": "text", + "text": "." + } + ] + }, + "type": { + "type": "reference", + "target": 7287, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "conditional", + "checkType": { + "type": "reference", + "target": 7287, + "name": "S", + "package": "cheerio", + "refersToTypeParameter": true + }, + "extendsType": { + "type": "intrinsic", + "name": "string" + }, + "trueType": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + "falseType": { + "type": "reference", + "target": 7286, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7290, + "name": "filterArray", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 398, + "character": 24 + } + ], + "signatures": [ + { + "id": 7291, + "name": "filterArray", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 398, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7292, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7293, + "name": "nodes", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": 7292, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + }, + { + "id": 7294, + "name": "match", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7292, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + }, + { + "id": 7295, + "name": "xmlMode", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + }, + { + "id": 7296, + "name": "root", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Document" + }, + "name": "Document", + "package": "domhandler" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": 7292, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + } + } + ] + }, + { + "id": 7233, + "name": "find", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 27, + "character": 24 + } + ], + "signatures": [ + { + "id": 7234, + "name": "find", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the descendants of each element in the current set of matched elements,\nfiltered by a selector, jQuery object, or element." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('#fruits').find('li').length;\n//=> 3\n$('#fruits').find($('.apple')).length;\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The found elements." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/find/", + "target": "https://api.jquery.com/find/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 27, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7235, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7236, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7235, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7237, + "name": "selectorOrHaystack", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to look for." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7311, + "name": "first", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 484, + "character": 24 + } + ], + "signatures": [ + { + "id": 7312, + "name": "first", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Will select the first element of a cheerio object." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('#fruits').children().first().text();\n//=> Apple\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The first element." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/first/", + "target": "https://api.jquery.com/first/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 484, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7313, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7314, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7313, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7313, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7324, + "name": "get", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 536, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 551, + "character": 24 + } + ], + "signatures": [ + { + "id": 7325, + "name": "get", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve one of the elements matched by the Cheerio object, at the " + }, + { + "kind": "code", + "text": "`i`" + }, + { + "kind": "text", + "text": "th\nposition." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').get(0).tagName;\n//=> li\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The element at the " + }, + { + "kind": "code", + "text": "`i`" + }, + { + "kind": "text", + "text": "th position." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/get/", + "target": "https://api.jquery.com/get/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 536, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7326, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7327, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7326, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7328, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to retrieve." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": 7326, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7329, + "name": "get", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve all elements matched by the Cheerio object, as an array." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').get().length;\n//=> 3\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "All elements matched by the Cheerio object." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/get/", + "target": "https://api.jquery.com/get/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 551, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7330, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7331, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7330, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": 7330, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + } + ] + }, + { + "id": 7307, + "name": "has", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 469, + "character": 24 + } + ], + "signatures": [ + { + "id": 7308, + "name": "has", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Filters the set of matched elements to only those which have the given DOM\nelement as a descendant or which have a descendant that matches the given\nselector. Equivalent to " + }, + { + "kind": "code", + "text": "`.filter(':has(selector)')`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "Selector", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').has('.pear').attr('id');\n//=> fruits\n```" + } + ] + }, + { + "tag": "@example", + "name": "Element", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').has($('.pear')[0]).attr('id');\n//=> fruits\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The filtered collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/has/", + "target": "https://api.jquery.com/has/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 469, + "character": 24 + } + ], + "parameters": [ + { + "id": 7309, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7310, + "name": "selectorOrHaystack", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to look for." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ] + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7193, + "name": "hasClass", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 321, + "character": 24 + } + ], + "signatures": [ + { + "id": 7194, + "name": "hasClass", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check to see if _any_ of the matched elements have the given " + }, + { + "kind": "code", + "text": "`className`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').hasClass('pear');\n//=> true\n\n$('apple').hasClass('fruit');\n//=> false\n\n$('li').hasClass('pear');\n//=> true\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "Indicates if an element has the given " + }, + { + "kind": "code", + "text": "`className`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/hasClass/", + "target": "https://api.jquery.com/hasClass/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 321, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7195, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7196, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7195, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7197, + "name": "className", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the class." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ] + }, + { + "id": 7491, + "name": "html", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 452, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 468, + "character": 24 + } + ], + "signatures": [ + { + "id": 7492, + "name": "html", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets an HTML content string from the first selected element." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').html();\n//=> Orange\n\n$('#fruits').html('
    • Mango
      • ').html();\n//=>
    • Mango
      • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The HTML content string." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/html/", + "target": "https://api.jquery.com/html/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 452, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7493, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7494, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7493, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "id": 7495, + "name": "html", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Replaces each selected element's content with the specified content." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').html('
    • Mango
      • ').html();\n//=>
    • Mango
      • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/html/", + "target": "https://api.jquery.com/html/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 468, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7496, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7497, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7496, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7498, + "name": "str", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content to replace selection's contents with." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7496, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7496, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7336, + "name": "index", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 583, + "character": 24 + } + ], + "signatures": [ + { + "id": 7337, + "name": "index", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Search for a given element from among the matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').index();\n//=> 2 $('.orange').index('li');\n//=> 1\n$('.apple').index($('#fruit, li'));\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The index of the element." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/index/", + "target": "https://api.jquery.com/index/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 583, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7338, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7339, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7338, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7340, + "name": "selectorOrNeedle", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to look for." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + ] + } + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + } + ] + }, + { + "id": 7457, + "name": "insertAfter", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 329, + "character": 24 + } + ], + "signatures": [ + { + "id": 7458, + "name": "insertAfter", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert every element in the set of matched elements after the target." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
    • Plum
      • ').insertAfter('.apple');\n$.html();\n//=>
        \n//
      • Apple
        • \n//
      • Plum
        • \n//
      • Orange
        • \n//
      • Pear
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The set of newly inserted elements." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/insertAfter/", + "target": "https://api.jquery.com/insertAfter/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 329, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7459, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7460, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7459, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7461, + "name": "target", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to insert elements after." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7459, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7472, + "name": "insertBefore", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 374, + "character": 24 + } + ], + "signatures": [ + { + "id": 7473, + "name": "insertBefore", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert every element in the set of matched elements before the target." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
    • Plum
      • ').insertBefore('.apple');\n$.html();\n//=>
        \n//
      • Plum
        • \n//
      • Apple
        • \n//
      • Orange
        • \n//
      • Pear
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The set of newly inserted elements." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/insertBefore/", + "target": "https://api.jquery.com/insertBefore/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 374, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7474, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7475, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7474, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7476, + "name": "target", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to insert elements before." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7474, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7297, + "name": "is", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 411, + "character": 24 + } + ], + "signatures": [ + { + "id": 7298, + "name": "is", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Checks the current list of elements and returns " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": " if _any_ of the\nelements match the selector. If using an element or Cheerio selection,\nreturns " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": " if _any_ of the elements match. If using a predicate function,\nthe function is executed in the context of the selected element, so " + }, + { + "kind": "code", + "text": "`this`" + }, + { + "kind": "text", + "text": "\nrefers to the current element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "Whether or not the selector matches an element of the instance." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/is/", + "target": "https://api.jquery.com/is/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 411, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7299, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7300, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7299, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7301, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for the selection." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7299, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ] + }, + { + "id": 7315, + "name": "last", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 499, + "character": 24 + } + ], + "signatures": [ + { + "id": 7316, + "name": "last", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Will select the last element of a cheerio object." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('#fruits').children().last().text();\n//=> Pear\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The last element." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/last/", + "target": "https://api.jquery.com/last/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 499, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7317, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7318, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7317, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7317, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7263, + "name": "map", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 336, + "character": 24 + } + ], + "signatures": [ + { + "id": 7264, + "name": "map", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pass each element in the current matched set through a function, producing a\nnew Cheerio object containing the return values. The function can return an\nindividual data item or an array of data items to be inserted into the\nresulting set. If an array is returned, the elements inside the array are\ninserted into the set. If the function returns null or undefined, no element\nwill be inserted." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li')\n .map(function (i, el) {\n // this === el\n return $(this).text();\n })\n .toArray()\n .join(' ');\n//=> \"apple orange pear\"\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The mapped elements, wrapped in a Cheerio collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/map/", + "target": "https://api.jquery.com/map/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 336, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7265, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + }, + { + "id": 7266, + "name": "M", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7267, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7265, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7268, + "name": "fn", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function to execute." + } + ] + }, + "type": { + "type": "reflection", + "declaration": { + "id": 7269, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 336, + "character": 56 + } + ], + "signatures": [ + { + "id": 7270, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 336, + "character": 56 + } + ], + "parameters": [ + { + "id": 7271, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7265, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7272, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7273, + "name": "el", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7265, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": 7266, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": 7266, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + } + } + ] + } + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7266, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7379, + "name": "next", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 136, + "character": 21 + } + ], + "signatures": [ + { + "id": 7380, + "name": "next", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets the next sibling of each selected element, optionally filtered by a\nselector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').next().hasClass('orange');\n//=> true\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The next nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/next/", + "target": "https://api.jquery.com/next/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 136, + "character": 27 + } + ], + "typeParameters": [ + { + "id": 7381, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7382, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7381, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7383, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for sibling." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7384, + "name": "nextAll", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 155, + "character": 21 + } + ], + "signatures": [ + { + "id": 7385, + "name": "nextAll", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets all the following siblings of the each selected element, optionally\nfiltered by a selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').nextAll();\n//=> [
    • Orange
      • ,
    • Pear
      • ]\n$('.apple').nextAll('.orange');\n//=> [
    • Orange
      • ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The next nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/nextAll/", + "target": "https://api.jquery.com/nextAll/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 155, + "character": 30 + } + ], + "typeParameters": [ + { + "id": 7386, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7387, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7386, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7388, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7389, + "name": "nextUntil", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 173, + "character": 21 + } + ], + "signatures": [ + { + "id": 7390, + "name": "nextUntil", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets all the following siblings up to but not including the element matched\nby the selector, optionally filtered by another selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple').nextUntil('.pear');\n//=> [
    • Orange
      • ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The next nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/nextUntil/", + "target": "https://api.jquery.com/nextUntil/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 173, + "character": 32 + } + ], + "typeParameters": [ + { + "id": 7391, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7392, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7391, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7393, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for element to stop at." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + ] + } + }, + { + "id": 7394, + "name": "filterSelector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7302, + "name": "not", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 444, + "character": 24 + } + ], + "signatures": [ + { + "id": 7303, + "name": "not", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove elements from the set of matched elements. Given a Cheerio object that\nrepresents a set of DOM elements, the " + }, + { + "kind": "code", + "text": "`.not()`" + }, + { + "kind": "text", + "text": " method constructs a new\nCheerio object from a subset of the matching elements. The supplied selector\nis tested against each element; the elements that don't match the selector\nwill be included in the result.\n\nThe " + }, + { + "kind": "code", + "text": "`.not()`" + }, + { + "kind": "text", + "text": " method can take a function as its argument in the same way that\n" + }, + { + "kind": "code", + "text": "`.filter()`" + }, + { + "kind": "text", + "text": " does. Elements for which the function returns " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": " are excluded\nfrom the filtered set; all other elements are included." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "Selector", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').not('.apple').length;\n//=> 2\n```" + } + ] + }, + { + "tag": "@example", + "name": "Function", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').not(function (i, el) {\n // this === el\n return $(this).attr('class') === 'orange';\n}).length; //=> 2\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The filtered collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/not/", + "target": "https://api.jquery.com/not/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 444, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7304, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7305, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7304, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7306, + "name": "match", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to look for, following the rules above." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7304, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7304, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7363, + "name": "parent", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 54, + "character": 21 + } + ], + "signatures": [ + { + "id": 7364, + "name": "parent", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the parent of each element in the current set of matched elements,\noptionally filtered by a selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').parent().attr('id');\n//=> fruits\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The parents." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/parent/", + "target": "https://api.jquery.com/parent/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 54, + "character": 29 + } + ], + "typeParameters": [ + { + "id": 7365, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7366, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7365, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7367, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for parent." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7368, + "name": "parents", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 73, + "character": 21 + } + ], + "signatures": [ + { + "id": 7369, + "name": "parents", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a set of parents filtered by " + }, + { + "kind": "code", + "text": "`selector`" + }, + { + "kind": "text", + "text": " of each element in the current\nset of match elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').parents().length;\n//=> 2\n$('.orange').parents('#fruits').length;\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The parents." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/parents/", + "target": "https://api.jquery.com/parents/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 73, + "character": 30 + } + ], + "typeParameters": [ + { + "id": 7370, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7371, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7370, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7372, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for parents." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7373, + "name": "parentsUntil", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 92, + "character": 21 + } + ], + "signatures": [ + { + "id": 7374, + "name": "parentsUntil", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the ancestors of each element in the current set of matched elements, up\nto but not including the element matched by the selector, DOM node, or\ncheerio object." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').parentsUntil('#food').length;\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The parents." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/parentsUntil/", + "target": "https://api.jquery.com/parentsUntil/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 92, + "character": 35 + } + ], + "typeParameters": [ + { + "id": 7375, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7376, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7375, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7377, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for element to stop at." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + ] + } + }, + { + "id": 7378, + "name": "filterSelector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional filter for parents." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7530, + "name": "prepend", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 104, + "character": 21 + } + ], + "signatures": [ + { + "id": 7531, + "name": "prepend", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Inserts content as the _first_ child of each of the selected elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('ul').prepend('
    • Plum
      • ');\n$.html();\n//=>
        \n//
      • Plum
        • \n//
      • Apple
        • \n//
      • Orange
        • \n//
      • Pear
        • \n//
      \n```" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prepend/", + "target": "https://api.jquery.com/prepend/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 104, + "character": 30 + } + ], + "typeParameters": [ + { + "id": 7532, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7533, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7532, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7534, + "name": "elems", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isRest": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + }, + { + "type": "tuple", + "elements": [ + { + "type": "reflection", + "declaration": { + "id": 7535, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 104, + "character": 79 + } + ], + "signatures": [ + { + "id": 7536, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 104, + "character": 79 + } + ], + "parameters": [ + { + "id": 7537, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7538, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7539, + "name": "html", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ] + } + } + ] + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7532, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7432, + "name": "prependTo", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 64, + "character": 24 + } + ], + "signatures": [ + { + "id": 7433, + "name": "prependTo", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Insert every element in the set of matched elements to the beginning of the\ntarget." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
    • Plum
      • ').prependTo('#fruits');\n$.html();\n//=>
        \n//
      • Plum
        • \n//
      • Apple
        • \n//
      • Orange
        • \n//
      • Pear
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prependTo/", + "target": "https://api.jquery.com/prependTo/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 64, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7434, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7435, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7434, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7436, + "name": "target", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to prepend elements to." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7434, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7395, + "name": "prev", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 190, + "character": 21 + } + ], + "signatures": [ + { + "id": 7396, + "name": "prev", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets the previous sibling of each selected element optionally filtered by a\nselector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').prev().hasClass('apple');\n//=> true\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The previous nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prev/", + "target": "https://api.jquery.com/prev/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 190, + "character": 27 + } + ], + "typeParameters": [ + { + "id": 7397, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7398, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7397, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7399, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7400, + "name": "prevAll", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 210, + "character": 21 + } + ], + "signatures": [ + { + "id": 7401, + "name": "prevAll", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets all the preceding siblings of each selected element, optionally filtered\nby a selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').prevAll();\n//=> [
    • Orange
      • ,
    • Apple
      • ]\n\n$('.pear').prevAll('.orange');\n//=> [
    • Orange
      • ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The previous nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prevAll/", + "target": "https://api.jquery.com/prevAll/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 210, + "character": 30 + } + ], + "typeParameters": [ + { + "id": 7402, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7403, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7402, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7404, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7405, + "name": "prevUntil", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 228, + "character": 21 + } + ], + "signatures": [ + { + "id": 7406, + "name": "prevUntil", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets all the preceding siblings up to but not including the element matched\nby the selector, optionally filtered by another selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').prevUntil('.apple');\n//=> [
    • Orange
      • ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The previous nodes." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prevUntil/", + "target": "https://api.jquery.com/prevUntil/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 228, + "character": 32 + } + ], + "typeParameters": [ + { + "id": 7407, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7408, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7407, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7409, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Selector for element to stop at." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + ] + } + }, + { + "id": 7410, + "name": "filterSelector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7096, + "name": "prop", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 103, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 104, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 112, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 127, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 134, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 142, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 158, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 166, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 173, + "character": 24 + } + ], + "signatures": [ + { + "id": 7097, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting and setting properties. Gets the property value for only\nthe first element in the matched set." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('input[type=\"checkbox\"]').prop('checked');\n//=> false\n\n$('input[type=\"checkbox\"]').prop('checked', true).val();\n//=> ok\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "If " + }, + { + "kind": "code", + "text": "`value`" + }, + { + "kind": "text", + "text": " is specified the instance itself, otherwise the prop's\n value." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/prop/", + "target": "https://api.jquery.com/prop/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 103, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7098, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7099, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7098, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7100, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": "tagName" + }, + { + "type": "literal", + "value": "nodeName" + } + ] + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7101, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 104, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7102, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7103, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7102, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7104, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": "innerHTML" + }, + { + "type": "literal", + "value": "outerHTML" + }, + { + "type": "literal", + "value": "innerText" + }, + { + "type": "literal", + "value": "textContent" + } + ] + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "id": 7105, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a parsed CSS style object." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The style object, or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": " if the element has no " + }, + { + "kind": "code", + "text": "`style`" + }, + { + "kind": "text", + "text": "\n attribute." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 112, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7106, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7107, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7106, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7108, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "literal", + "value": "style" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/api/attributes.ts", + "qualifiedName": "StyleProp" + }, + "name": "StyleProp", + "package": "cheerio" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7109, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Resolve " + }, + { + "kind": "code", + "text": "`href`" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "code", + "text": "`src`" + }, + { + "kind": "text", + "text": " of supported elements. Requires the " + }, + { + "kind": "code", + "text": "`baseURI`" + }, + { + "kind": "text", + "text": " option\nto be set, and a global " + }, + { + "kind": "code", + "text": "`URL`" + }, + { + "kind": "text", + "text": " object to be part of the environment." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "With `baseURI` set to `'https://example.com'`:", + "content": [ + { + "kind": "code", + "text": "```js\n$('').prop('src');\n//=> 'https://example.com/image.png'\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The resolved URL, or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": " if the element is not supported." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 127, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7110, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7111, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7110, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7112, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": "href" + }, + { + "type": "literal", + "value": "src" + } + ] + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + } + }, + { + "id": 7113, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a property of an element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The property's value." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 134, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7114, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7115, + "name": "K", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "typeOperator", + "operator": "keyof", + "target": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + } + } + ], + "parameters": [ + { + "id": 7116, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7114, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7117, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "reference", + "target": 7115, + "name": "K", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "indexedAccess", + "indexType": { + "type": "reference", + "target": 7115, + "name": "K", + "package": "cheerio", + "refersToTypeParameter": true + }, + "objectType": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + } + }, + { + "id": 7118, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a property of an element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 142, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7119, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7120, + "name": "K", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "typeOperator", + "operator": "keyof", + "target": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + } + } + ], + "parameters": [ + { + "id": 7121, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7119, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7122, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "reference", + "target": 7120, + "name": "K", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7123, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set the property to." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "indexedAccess", + "indexType": { + "type": "reference", + "target": 7120, + "name": "K", + "package": "cheerio", + "refersToTypeParameter": true + }, + "objectType": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "type": "reflection", + "declaration": { + "id": 7124, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 142, + "character": 121 + } + ], + "signatures": [ + { + "id": 7125, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 142, + "character": 121 + } + ], + "parameters": [ + { + "id": 7126, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7127, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7128, + "name": "prop", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7120, + "name": "K", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "number" + }, + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Document" + }, + "name": "Document", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "CDATA" + }, + "name": "CDATA", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Text" + }, + "name": "Text", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Comment" + }, + "name": "Comment", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ProcessingInstruction" + }, + "name": "ProcessingInstruction", + "package": "domhandler" + }, + { + "type": "reflection", + "declaration": { + "id": 7129, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 169, + "character": 13 + } + ], + "indexSignatures": [ + { + "id": 7130, + "name": "__index", + "variant": "signature", + "kind": 8192, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 170, + "character": 8 + } + ], + "parameters": [ + { + "id": 7131, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "TagSourceCodeLocation" + }, + "name": "TagSourceCodeLocation", + "package": "domhandler" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Attribute" + }, + "name": "Attribute", + "package": "domhandler" + } + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + }, + { + "type": "reflection", + "declaration": { + "id": 7132, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "signatures": [ + { + "id": 7133, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone this node, and optionally its children." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "A clone of the node." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "typeParameters": [ + { + "id": 7134, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Node" + }, + "name": "Node", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7135, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + } + }, + { + "id": 7136, + "name": "recursive", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone child nodes as well." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + } + } + ] + } + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7119, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7137, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set multiple properties of an element." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('input[type=\"checkbox\"]').prop({\n checked: true,\n disabled: false,\n});\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 158, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7138, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7139, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7138, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7140, + "name": "map", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Object of properties to set." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "number" + }, + { + "type": "intrinsic", + "name": "boolean" + }, + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Document" + }, + "name": "Document", + "package": "domhandler" + }, + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "CDATA" + }, + "name": "CDATA", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Text" + }, + "name": "Text", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Comment" + }, + "name": "Comment", + "package": "domhandler" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ProcessingInstruction" + }, + "name": "ProcessingInstruction", + "package": "domhandler" + }, + { + "type": "reflection", + "declaration": { + "id": 7141, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 169, + "character": 13 + } + ], + "indexSignatures": [ + { + "id": 7142, + "name": "__index", + "variant": "signature", + "kind": 8192, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 170, + "character": 8 + } + ], + "parameters": [ + { + "id": 7143, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "TagSourceCodeLocation" + }, + "name": "TagSourceCodeLocation", + "package": "domhandler" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Attribute" + }, + "name": "Attribute", + "package": "domhandler" + } + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + }, + { + "type": "reflection", + "declaration": { + "id": 7144, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "signatures": [ + { + "id": 7145, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone this node, and optionally its children." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "A clone of the node." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "typeParameters": [ + { + "id": 7146, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Node" + }, + "name": "Node", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7147, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + } + }, + { + "id": 7148, + "name": "recursive", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone child nodes as well." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + } + } + ] + } + } + ] + } + ], + "name": "Record", + "package": "typescript" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7138, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7149, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a property of an element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 166, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7150, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7151, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7150, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7152, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the property." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7153, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set the property to." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "boolean" + }, + { + "type": "reflection", + "declaration": { + "id": 7154, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 166, + "character": 114 + } + ], + "signatures": [ + { + "id": 7155, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 166, + "character": 114 + } + ], + "parameters": [ + { + "id": 7156, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7157, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7158, + "name": "prop", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "boolean" + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7150, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7159, + "name": "prop", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a property of an element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The property's value." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 173, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7160, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7161, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7160, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7162, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The property's name." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + }, + { + "id": 7477, + "name": "remove", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 395, + "character": 24 + } + ], + "signatures": [ + { + "id": 7478, + "name": "remove", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Removes the set of matched elements from the DOM and all their children.\n" + }, + { + "kind": "code", + "text": "`selector`" + }, + { + "kind": "text", + "text": " filters the set of matched elements to be removed." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').remove();\n$.html();\n//=>
        \n//
      • Apple
        • \n//
      • Orange
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/remove/", + "target": "https://api.jquery.com/remove/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 395, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7479, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7480, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7479, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7481, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional selector for elements to remove." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7479, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7188, + "name": "removeAttr", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 299, + "character": 24 + } + ], + "signatures": [ + { + "id": 7189, + "name": "removeAttr", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for removing attributes by " + }, + { + "kind": "code", + "text": "`name`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').removeAttr('class').html();\n//=>
    • Pear
      • \n\n$('.apple').attr('id', 'favorite');\n$('.apple').removeAttr('id class').html();\n//=>
    • Apple
      • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/removeAttr/", + "target": "https://api.jquery.com/removeAttr/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 299, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7190, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7191, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7190, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7192, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the attribute." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7190, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7209, + "name": "removeClass", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 361, + "character": 24 + } + ], + "signatures": [ + { + "id": 7210, + "name": "removeClass", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Removes one or more space-separated classes from the selected elements. If no\n" + }, + { + "kind": "code", + "text": "`className`" + }, + { + "kind": "text", + "text": " is defined, all classes will be removed. Also accepts a\n" + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').removeClass('pear').html();\n//=>
    • Pear
      • \n\n$('.apple').addClass('red').removeClass().html();\n//=>
    • Apple
      • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/removeClass/", + "target": "https://api.jquery.com/removeClass/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 361, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7211, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7212, + "name": "R", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7211, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + } + ], + "parameters": [ + { + "id": 7213, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7212, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7214, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the class. If not specified, removes all elements." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7215, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 361, + "character": 105 + } + ], + "signatures": [ + { + "id": 7216, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 361, + "character": 105 + } + ], + "parameters": [ + { + "id": 7217, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7218, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7219, + "name": "className", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "intrinsic", + "name": "string" + } + ] + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7212, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + }, + { + "id": 7482, + "name": "replaceWith", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 417, + "character": 24 + } + ], + "signatures": [ + { + "id": 7483, + "name": "replaceWith", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Replaces matched elements with " + }, + { + "kind": "code", + "text": "`content`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst plum = $('
    • Plum
      • ');\n$('.pear').replaceWith(plum);\n$.html();\n//=>
        \n//
      • Apple
        • \n//
      • Orange
        • \n//
      • Plum
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/replaceWith/", + "target": "https://api.jquery.com/replaceWith/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 417, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7484, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7485, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7484, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7486, + "name": "content", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Replacement for matched elements." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "AcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7484, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7573, + "name": "serialize", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 17, + "character": 24 + } + ], + "signatures": [ + { + "id": 7574, + "name": "serialize", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encode a set of form elements as a string for submission." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
      ').serialize();\n//=> 'foo=bar'\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The serialized form." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/serialize/", + "target": "https://api.jquery.com/serialize/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 17, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7575, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7576, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7575, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + }, + { + "id": 7577, + "name": "serializeArray", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 32, + "character": 24 + } + ], + "signatures": [ + { + "id": 7578, + "name": "serializeArray", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encode a set of form elements as an array of names and values." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('
      ').serializeArray();\n//=> [ { name: 'foo', value: 'bar' } ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The serialized form." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/serializeArray/", + "target": "https://api.jquery.com/serializeArray/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 32, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7579, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7580, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7579, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reflection", + "declaration": { + "id": 7581, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "children": [ + { + "id": 7582, + "name": "name", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 33, + "character": 4 + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7583, + "name": "value", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 34, + "character": 4 + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7582, + 7583 + ] + } + ], + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/forms.d.ts", + "line": 32, + "character": 77 + } + ] + } + } + } + } + ] + }, + { + "id": 7411, + "name": "siblings", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 248, + "character": 21 + } + ], + "signatures": [ + { + "id": 7412, + "name": "siblings", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the siblings of each element (excluding the element) in the set of\nmatched elements, optionally filtered by a selector." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.pear').siblings().length;\n//=> 2\n\n$('.pear').siblings('.orange').length;\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The siblings." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/siblings/", + "target": "https://api.jquery.com/siblings/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 248, + "character": 31 + } + ], + "typeParameters": [ + { + "id": 7413, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7414, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7413, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7415, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified filter for siblings." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedFilters" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "AcceptedFilters", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7341, + "name": "slice", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 606, + "character": 24 + } + ], + "signatures": [ + { + "id": 7342, + "name": "slice", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Gets the elements matching the specified range (0-based position)." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').slice(1).eq(0).text();\n//=> 'Orange'\n\n$('li').slice(1, 2).length;\n//=> 1\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The elements matching the specified range." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/slice/", + "target": "https://api.jquery.com/slice/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 606, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7343, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7344, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7343, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7345, + "name": "start", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A position at which the elements begin to be selected. If\n negative, it indicates an offset from the end of the set." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7346, + "name": "end", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A position at which the elements stop being selected. If\n negative, it indicates an offset from the end of the set. If omitted, the\n range continues until the end of the set." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "number" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7343, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7503, + "name": "text", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 496, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 513, + "character": 24 + } + ], + "signatures": [ + { + "id": 7504, + "name": "text", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the combined text contents of each element in the set of matched\nelements, including their descendants." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').text();\n//=> Orange\n\n$('ul').text();\n//=> Apple\n// Orange\n// Pear\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The text contents of the collection." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/text/", + "target": "https://api.jquery.com/text/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 496, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7505, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7506, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7505, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7507, + "name": "text", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the content of each element in the set of matched elements to the\nspecified text." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.orange').text('Orange');\n//=>
      Orange
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/text/", + "target": "https://api.jquery.com/text/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 513, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7508, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7509, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7508, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7510, + "name": "str", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The text to set as the content of each matched element." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7511, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 513, + "character": 81 + } + ], + "signatures": [ + { + "id": 7512, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 513, + "character": 81 + } + ], + "parameters": [ + { + "id": 7513, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7514, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7515, + "name": "text", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7508, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7332, + "name": "toArray", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 564, + "character": 24 + } + ], + "signatures": [ + { + "id": 7333, + "name": "toArray", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve all the DOM elements contained in the jQuery set as an array." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('li').toArray();\n//=> [ {...}, {...}, {...} ]\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The contained items." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/traversing.d.ts", + "line": 564, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7334, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7335, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7334, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": 7334, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + } + } + ] + }, + { + "id": 7220, + "name": "toggleClass", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 383, + "character": 24 + } + ], + "signatures": [ + { + "id": 7221, + "name": "toggleClass", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add or remove class(es) from the matched elements, depending on either the\nclass's presence or the value of the switch argument. Also accepts a\n" + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('.apple.green').toggleClass('fruit green red').html();\n//=>
    • Apple
      • \n\n$('.apple.green').toggleClass('fruit green red', true).html();\n//=>
    • Apple
      • \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/toggleClass/", + "target": "https://api.jquery.com/toggleClass/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 383, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7222, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7223, + "name": "R", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7222, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + } + ], + "parameters": [ + { + "id": 7224, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7223, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + }, + { + "id": 7225, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the class. Can also be a function." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reflection", + "declaration": { + "id": 7226, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 383, + "character": 106 + } + ], + "signatures": [ + { + "id": 7227, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 383, + "character": 106 + } + ], + "parameters": [ + { + "id": 7228, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + } + }, + { + "id": 7229, + "name": "i", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7230, + "name": "className", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7231, + "name": "stateVal", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + } + ] + } + }, + { + "id": 7232, + "name": "stateVal", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If specified the state of the class." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "reference", + "target": 7223, + "name": "R", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ] + }, + { + "id": 7499, + "name": "toString", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 475, + "character": 24 + } + ], + "signatures": [ + { + "id": 7500, + "name": "toString", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Turns the collection to a string. Alias for " + }, + { + "kind": "code", + "text": "`.html()`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The rendered document." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 475, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7501, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7502, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7501, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + }, + { + "id": 7437, + "name": "unwrap", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 232, + "character": 24 + } + ], + "signatures": [ + { + "id": 7438, + "name": "unwrap", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The .unwrap() function, removes the parents of the set of matched elements\nfrom the DOM, leaving the matched elements in their place." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "without selector", + "content": [ + { + "kind": "code", + "text": "```js\nconst $ = cheerio.load(\n '
      \\n

      Hello

      \\n

      World

      \\n
      ',\n);\n$('#test p').unwrap();\n\n//=>
      \n//

      Hello

      \n//

      World

      \n//
      \n```" + } + ] + }, + { + "tag": "@example", + "name": "with selector", + "content": [ + { + "kind": "code", + "text": "```js\nconst $ = cheerio.load(\n '
      \\n

      Hello

      \\n

      World

      \\n
      ',\n);\n$('#test p').unwrap('b');\n\n//=>
      \n//

      Hello

      \n//

      World

      \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself, for chaining." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/unwrap/", + "target": "https://api.jquery.com/unwrap/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 232, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7439, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7440, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7439, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7441, + "name": "selector", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A selector to check the parent element against. If an\n element's parent does not match the selector, the element won't be\n unwrapped." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7439, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7180, + "name": "val", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 262, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 279, + "character": 24 + } + ], + "signatures": [ + { + "id": 7181, + "name": "val", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for getting the value of input, select, and textarea. Note: Support\nfor " + }, + { + "kind": "code", + "text": "`map`" + }, + { + "kind": "text", + "text": ", and " + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": " has not been added yet." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('input[type=\"text\"]').val();\n//=> input_text\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The value." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/val/", + "target": "https://api.jquery.com/val/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 262, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7182, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7183, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7182, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "undefined" + }, + { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + }, + { + "id": 7184, + "name": "val", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Method for setting the value of input, select, and textarea. Note: Support\nfor " + }, + { + "kind": "code", + "text": "`map`" + }, + { + "kind": "text", + "text": ", and " + }, + { + "kind": "code", + "text": "`function`" + }, + { + "kind": "text", + "text": " has not been added yet." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$('input[type=\"text\"]').val('test').html();\n//=> \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/val/", + "target": "https://api.jquery.com/val/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/attributes.d.ts", + "line": 279, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7185, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7186, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7185, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7187, + "name": "value", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The new value." + } + ] + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7185, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7540, + "name": "wrap", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 148, + "character": 21 + } + ], + "signatures": [ + { + "id": 7541, + "name": "wrap", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The .wrap() function can take any string or object that could be passed to\nthe $() factory function to specify a DOM structure. This structure may be\nnested several levels deep, but should contain only one inmost element. A\ncopy of this structure will be wrapped around each of the elements in the set\nof matched elements. This method returns the original set of elements for\nchaining purposes." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst redFruit = $('
      ');\n$('.apple').wrap(redFruit);\n\n//=>
        \n//
        \n//
      • Apple
        • \n//
        \n//
      • Orange
        • \n//
      • Plum
        • \n//
      \n\nconst healthy = $('
      ');\n$('li').wrap(healthy);\n\n//=>
        \n//
        \n//
      • Apple
        • \n//
        \n//
        \n//
      • Orange
        • \n//
        \n//
        \n//
      • Plum
        • \n//
        \n//
      \n```" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/wrap/", + "target": "https://api.jquery.com/wrap/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 148, + "character": 27 + } + ], + "typeParameters": [ + { + "id": 7542, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7543, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7542, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7544, + "name": "wrapper", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The DOM structure to wrap around each element in the\n selection." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "AcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7542, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7442, + "name": "wrapAll", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 284, + "character": 24 + } + ], + "signatures": [ + { + "id": 7443, + "name": "wrapAll", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The .wrapAll() function can take any string or object that could be passed to\nthe $() function to specify a DOM structure. This structure may be nested\nseveral levels deep, but should contain only one inmost element. The\nstructure will be wrapped around all of the elements in the set of matched\nelements, as a single group." + } + ], + "blockTags": [ + { + "tag": "@example", + "name": "With markup passed to `wrapAll`", + "content": [ + { + "kind": "code", + "text": "```js\nconst $ = cheerio.load(\n '
      First
      Second
      ',\n);\n$('.inner').wrapAll(\"
      \");\n\n//=>
      \n//
      \n//
      First
      \n//
      Second
      \n//
      \n//
      \n```" + } + ] + }, + { + "tag": "@example", + "name": "With an existing cheerio instance", + "content": [ + { + "kind": "code", + "text": "```js\nconst $ = cheerio.load(\n 'Span 1StrongSpan 2',\n);\nconst wrap = $('

      ');\n$('span').wrapAll(wrap);\n\n//=>
      \n//

      \n// \n// \n// Span 1\n// Span 2\n// \n// \n//

      \n//
      \n// Strong\n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/wrapAll/", + "target": "https://api.jquery.com/wrapAll/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 284, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7444, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7445, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7444, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7446, + "name": "wrapper", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The DOM structure to wrap around all matched elements in the\n selection." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7444, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "AcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7444, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + }, + { + "id": 7545, + "name": "wrapInner", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 192, + "character": 21 + } + ], + "signatures": [ + { + "id": 7546, + "name": "wrapInner", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The .wrapInner() function can take any string or object that could be passed\nto the $() factory function to specify a DOM structure. This structure may be\nnested several levels deep, but should contain only one inmost element. The\nstructure will be wrapped around the content of each of the elements in the\nset of matched elements." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst redFruit = $('
      ');\n$('.apple').wrapInner(redFruit);\n\n//=>
        \n//
      • \n//
        Apple
        \n//
        • \n//
      • Orange
        • \n//
      • Pear
        • \n//
      \n\nconst healthy = $('
      ');\n$('li').wrapInner(healthy);\n\n//=>
        \n//
      • \n//
        Apple
        \n//
        • \n//
      • \n//
        Orange
        \n//
        • \n//
      • \n//
        Pear
        \n//
        • \n//
      \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The instance itself, for chaining." + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/wrapInner/", + "target": "https://api.jquery.com/wrapInner/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/api/manipulation.d.ts", + "line": 192, + "character": 32 + } + ], + "typeParameters": [ + { + "id": 7547, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7548, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7547, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 7549, + "name": "wrapper", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The DOM structure to wrap around the content of each element\n in the selection." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "AcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "AcceptedElems", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": 7547, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "Cheerio", + "package": "cheerio" + } + } + ] + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7064, + 7046, + 7047, + 7049, + 7065 + ] + }, + { + "title": "Methods", + "children": [ + 7590, + 7351, + 7358, + 7198, + 7447, + 7520, + 7427, + 7074, + 7462, + 7416, + 7516, + 7244, + 7249, + 7550, + 7163, + 7253, + 7487, + 7347, + 7319, + 7584, + 7274, + 7290, + 7233, + 7311, + 7324, + 7307, + 7193, + 7491, + 7336, + 7457, + 7472, + 7297, + 7315, + 7263, + 7379, + 7384, + 7389, + 7302, + 7363, + 7368, + 7373, + 7530, + 7432, + 7395, + 7400, + 7405, + 7096, + 7477, + 7188, + 7209, + 7482, + 7573, + 7577, + 7411, + 7341, + 7503, + 7332, + 7220, + 7499, + 7437, + 7180, + 7540, + 7442, + 7545 + ] + } + ], + "categories": [ + { + "title": "Attributes", + "children": [ + 7198, + 7074, + 7163, + 7193, + 7096, + 7188, + 7209, + 7220, + 7180 + ] + }, + { + "title": "CSS", + "children": [ + 7550 + ] + }, + { + "title": "Forms", + "children": [ + 7573, + 7577 + ] + }, + { + "title": "Manipulation", + "children": [ + 7447, + 7520, + 7427, + 7462, + 7516, + 7487, + 7491, + 7457, + 7472, + 7530, + 7432, + 7477, + 7482, + 7503, + 7499, + 7437, + 7540, + 7442, + 7545 + ] + }, + { + "title": "Traversing", + "children": [ + 7351, + 7358, + 7416, + 7244, + 7249, + 7253, + 7347, + 7319, + 7274, + 7233, + 7311, + 7324, + 7307, + 7336, + 7297, + 7315, + 7263, + 7379, + 7384, + 7389, + 7302, + 7363, + 7368, + 7373, + 7395, + 7400, + 7405, + 7411, + 7341 + ] + }, + { + "title": "__CATEGORY__", + "children": [ + 7064, + 7046, + 7047, + 7049, + 7065, + 7590, + 7584, + 7290, + 7332 + ] + } + ], + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 30, + "character": 30 + }, + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 80, + "character": 17 + } + ], + "typeParameters": [ + { + "id": 7592, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "indexSignatures": [ + { + "id": 7593, + "name": "__index", + "variant": "signature", + "kind": 8192, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/cheerio.d.ts", + "line": 32, + "character": 4 + } + ], + "parameters": [ + { + "id": 7594, + "name": "index", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "number" + } + } + ], + "type": { + "type": "reference", + "target": 7592, + "name": "T", + "package": "cheerio", + "qualifiedName": "Cheerio.T", + "refersToTypeParameter": true + } + } + ], + "extendedTypes": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/cheerio.ts", + "qualifiedName": "MethodsType" + }, + "name": "MethodsType", + "package": "cheerio" + }, + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es2015.iterable.d.ts", + "qualifiedName": "Iterable" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7592, + "name": "T", + "package": "cheerio", + "qualifiedName": "Cheerio.T", + "refersToTypeParameter": true + } + ], + "name": "Iterable", + "package": "typescript" + } + ], + "implementedTypes": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7592, + "name": "T", + "package": "cheerio", + "qualifiedName": "Cheerio.T", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + ] + }, + { + "id": 7595, + "name": "Element", + "variant": "declaration", + "kind": 128, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An element within the DOM." + } + ] + }, + "children": [ + { + "id": 7596, + "name": "constructor", + "variant": "declaration", + "kind": 512, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 178, + "character": 4 + } + ], + "signatures": [ + { + "id": 7597, + "name": "new Element", + "variant": "signature", + "kind": 16384, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 178, + "character": 4 + } + ], + "parameters": [ + { + "id": 7598, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the tag, eg. " + }, + { + "kind": "code", + "text": "`div`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`span`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7599, + "name": "attribs", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Object mapping attribute names to attribute values." + } + ] + }, + "type": { + "type": "reflection", + "declaration": { + "id": 7600, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 178, + "character": 39 + } + ], + "indexSignatures": [ + { + "id": 7601, + "name": "__index", + "variant": "signature", + "kind": 8192, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 179, + "character": 8 + } + ], + "parameters": [ + { + "id": 7602, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + } + }, + { + "id": 7603, + "name": "children", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Children of the node." + } + ] + }, + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + } + }, + { + "id": 7604, + "name": "type", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Script" + }, + "name": "Script", + "package": "domelementtype", + "qualifiedName": "ElementType.Script" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Style" + }, + "name": "Style", + "package": "domelementtype", + "qualifiedName": "ElementType.Style" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Tag" + }, + "name": "Tag", + "package": "domelementtype", + "qualifiedName": "ElementType.Tag" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": 7595, + "name": "Element", + "package": "domhandler" + }, + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.constructor" + } + } + ], + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.constructor" + } + }, + { + "id": 7606, + "name": "attribs", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 169, + "character": 4 + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7607, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 169, + "character": 13 + } + ], + "indexSignatures": [ + { + "id": 7608, + "name": "__index", + "variant": "signature", + "kind": 8192, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 170, + "character": 8 + } + ], + "parameters": [ + { + "id": 7609, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ] + } + } + }, + { + "id": 7623, + "name": "children", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 126, + "character": 4 + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.children" + } + }, + { + "id": 7636, + "name": "endIndex", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The end index of the node. Requires " + }, + { + "kind": "code", + "text": "`withEndIndices`" + }, + { + "kind": "text", + "text": " on the handler to be `true." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 39, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "number" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.endIndex" + } + }, + { + "id": 7605, + "name": "name", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 168, + "character": 4 + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7620, + "name": "namespace", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element namespace (parse5 only)." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 196, + "character": 4 + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7634, + "name": "next", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Next sibling" + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 35, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.next" + } + }, + { + "id": 7632, + "name": "parent", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Parent of the node" + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 31, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ParentNode" + }, + "name": "ParentNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.parent" + } + }, + { + "id": 7633, + "name": "prev", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Previous sibling" + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 33, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.prev" + } + }, + { + "id": 7613, + "name": "sourceCodeLocation", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "code", + "text": "`parse5`" + }, + { + "kind": "text", + "text": " source code location info, with start & end tags.\n\nAvailable if parsing with parse5 and location info is enabled." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 187, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "TagSourceCodeLocation" + }, + "name": "TagSourceCodeLocation", + "package": "domhandler" + } + ] + }, + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.sourceCodeLocation" + } + }, + { + "id": 7635, + "name": "startIndex", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The start index of the node. Requires " + }, + { + "kind": "code", + "text": "`withStartIndices`" + }, + { + "kind": "text", + "text": " on the handler to be `true." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 37, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "intrinsic", + "name": "number" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.startIndex" + } + }, + { + "id": 7610, + "name": "type", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The type of the node." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 172, + "character": 4 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Script" + }, + "name": "Script", + "package": "domelementtype", + "qualifiedName": "ElementType.Script" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Style" + }, + "name": "Style", + "package": "domelementtype", + "qualifiedName": "ElementType.Style" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domelementtype/src/index.ts", + "qualifiedName": "ElementType.Tag" + }, + "name": "Tag", + "package": "domelementtype", + "qualifiedName": "ElementType.Tag" + } + ] + }, + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.type" + } + }, + { + "id": 7621, + "name": "x-attribsNamespace", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element attribute namespaces (parse5 only)." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 198, + "character": 4 + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + } + }, + { + "id": 7622, + "name": "x-attribsPrefix", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element attribute namespace-related prefixes (parse5 only)." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 200, + "character": 4 + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Record" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "intrinsic", + "name": "string" + } + ], + "name": "Record", + "package": "typescript" + } + }, + { + "id": 7618, + "name": "attributes", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 194, + "character": 8 + } + ], + "getSignature": { + "id": 7619, + "name": "attributes", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 194, + "character": 8 + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Attribute" + }, + "name": "Attribute", + "package": "domhandler" + } + } + } + }, + { + "id": 7628, + "name": "childNodes", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 139, + "character": 8 + }, + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 140, + "character": 8 + } + ], + "getSignature": { + "id": 7629, + "name": "childNodes", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Same as " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "children", + "target": 7623 + }, + { + "kind": "text", + "text": ".\n[DOM spec](https://dom.spec.whatwg.org)-compatible alias." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 139, + "character": 8 + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.childNodes" + } + }, + "setSignature": { + "id": 7630, + "name": "childNodes", + "variant": "signature", + "kind": 1048576, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 140, + "character": 8 + } + ], + "parameters": [ + { + "id": 7631, + "name": "children", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.childNodes" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.childNodes" + } + }, + { + "id": 7624, + "name": "firstChild", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 132, + "character": 8 + } + ], + "getSignature": { + "id": 7625, + "name": "firstChild", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "First child of the node." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 132, + "character": 8 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.firstChild" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.firstChild" + } + }, + { + "id": 7626, + "name": "lastChild", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 134, + "character": 8 + } + ], + "getSignature": { + "id": 7627, + "name": "lastChild", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Last child of the node." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 134, + "character": 8 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.lastChild" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.lastChild" + } + }, + { + "id": 7645, + "name": "nextSibling", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 67, + "character": 8 + }, + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 68, + "character": 8 + } + ], + "getSignature": { + "id": 7646, + "name": "nextSibling", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Same as " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "next", + "target": 7634 + }, + { + "kind": "text", + "text": ".\n[DOM spec](https://dom.spec.whatwg.org)-compatible alias." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 67, + "character": 8 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.nextSibling" + } + }, + "setSignature": { + "id": 7647, + "name": "nextSibling", + "variant": "signature", + "kind": 1048576, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 68, + "character": 8 + } + ], + "parameters": [ + { + "id": 7648, + "name": "next", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.nextSibling" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.nextSibling" + } + }, + { + "id": 7611, + "name": "nodeType", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 181, + "character": 8 + } + ], + "getSignature": { + "id": 7612, + "name": "nodeType", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "[DOM spec](https://dom.spec.whatwg.org/#dom-node-nodetype)-compatible\nnode " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "type", + "target": 7610 + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 181, + "character": 8 + } + ], + "type": { + "type": "literal", + "value": 1 + }, + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.nodeType" + } + }, + "overwrites": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.nodeType" + } + }, + { + "id": 7637, + "name": "parentNode", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 55, + "character": 8 + }, + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 56, + "character": 8 + } + ], + "getSignature": { + "id": 7638, + "name": "parentNode", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Same as " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "parent", + "target": 7632 + }, + { + "kind": "text", + "text": ".\n[DOM spec](https://dom.spec.whatwg.org)-compatible alias." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 55, + "character": 8 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ParentNode" + }, + "name": "ParentNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.parentNode" + } + }, + "setSignature": { + "id": 7639, + "name": "parentNode", + "variant": "signature", + "kind": 1048576, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 56, + "character": 8 + } + ], + "parameters": [ + { + "id": 7640, + "name": "parent", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ParentNode" + }, + "name": "ParentNode", + "package": "domhandler" + } + ] + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.parentNode" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.parentNode" + } + }, + { + "id": 7641, + "name": "previousSibling", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 61, + "character": 8 + }, + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 62, + "character": 8 + } + ], + "getSignature": { + "id": 7642, + "name": "previousSibling", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Same as " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "prev", + "target": 7633 + }, + { + "kind": "text", + "text": ".\n[DOM spec](https://dom.spec.whatwg.org)-compatible alias." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 61, + "character": 8 + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.previousSibling" + } + }, + "setSignature": { + "id": 7643, + "name": "previousSibling", + "variant": "signature", + "kind": 1048576, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 62, + "character": 8 + } + ], + "parameters": [ + { + "id": 7644, + "name": "prev", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "ChildNode" + }, + "name": "ChildNode", + "package": "domhandler" + } + ] + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.previousSibling" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.previousSibling" + } + }, + { + "id": 7614, + "name": "tagName", + "variant": "declaration", + "kind": 262144, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 192, + "character": 8 + }, + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 193, + "character": 8 + } + ], + "getSignature": { + "id": 7615, + "name": "tagName", + "variant": "signature", + "kind": 524288, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Same as " + }, + { + "kind": "inline-tag", + "tag": "@link", + "text": "name", + "target": 7605 + }, + { + "kind": "text", + "text": ".\n[DOM spec](https://dom.spec.whatwg.org)-compatible alias." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 192, + "character": 8 + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + "setSignature": { + "id": 7616, + "name": "tagName", + "variant": "signature", + "kind": 1048576, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 193, + "character": 8 + } + ], + "parameters": [ + { + "id": 7617, + "name": "name", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + } + } + }, + { + "id": 7649, + "name": "cloneNode", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "signatures": [ + { + "id": 7650, + "name": "cloneNode", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone this node, and optionally its children." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "A clone of the node." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 75, + "character": 4 + } + ], + "typeParameters": [ + { + "id": 7651, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Node" + }, + "name": "Node", + "package": "domhandler" + } + } + ], + "parameters": [ + { + "id": 7652, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + } + }, + { + "id": 7653, + "name": "recursive", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Clone child nodes as well." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "reference", + "target": 7134, + "name": "T", + "package": "domhandler", + "refersToTypeParameter": true + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.cloneNode" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "NodeWithChildren.cloneNode" + } + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 7596 + ] + }, + { + "title": "Properties", + "children": [ + 7606, + 7623, + 7636, + 7605, + 7620, + 7634, + 7632, + 7633, + 7613, + 7635, + 7610, + 7621, + 7622 + ] + }, + { + "title": "Accessors", + "children": [ + 7618, + 7628, + 7624, + 7626, + 7645, + 7611, + 7637, + 7641, + 7614 + ] + }, + { + "title": "Methods", + "children": [ + 7649 + ] + } + ], + "sources": [ + { + "fileName": "node_modules/domhandler/lib/esm/node.d.ts", + "line": 167, + "character": 21 + } + ], + "extendedTypes": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "NodeWithChildren" + }, + "name": "NodeWithChildren", + "package": "domhandler" + } + ] + }, + { + "id": 7930, + "name": "BasicCrawlerOptions", + "variant": "declaration", + "kind": 256, + "flags": {}, + "children": [ + { + "id": 7950, + "name": "autoscaledPoolOptions", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom options passed to the underlying " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPool" + }, + { + "kind": "text", + "text": " constructor.\n> *NOTE:* The " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`" + }, + { + "kind": "text", + "text": "\noption is provided by the crawler and cannot be overridden.\nHowever, we can provide custom implementations of " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`" + }, + { + "kind": "text", + "text": "\nand " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 277, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L277" + } + ], + "type": { + "type": "reference", + "target": 243, + "name": "AutoscaledPoolOptions", + "package": "@crawlee/core" + } + }, + { + "id": 7936, + "name": "contextPipelineBuilder", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "*Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the " + }, + { + "kind": "code", + "text": "`Context`" + }, + { + "kind": "text", + "text": " type parameter.\n\nThe option is not required if your crawler subclass does not extend the crawling context with custom information or helpers." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 177, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L177" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7937, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 177, + "character": 29, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L177" + } + ], + "signatures": [ + { + "id": 7938, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 177, + "character": 29, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L177" + } + ], + "type": { + "type": "reference", + "target": 669, + "typeArguments": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 7970, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.Context", + "refersToTypeParameter": true + } + ], + "name": "ContextPipeline", + "package": "@crawlee/core" + } + } + ] + } + } + }, + { + "id": 7943, + "name": "errorHandler", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "User-provided function that allows modifying the request object before it gets retried by the crawler.\nIt's executed before each retry for the requests that failed less than " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`" + }, + { + "kind": "text", + "text": " times.\n\nThe function receives the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext" + }, + { + "kind": "text", + "text": " as the first argument,\nwhere the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext.request|`request`" + }, + { + "kind": "text", + "text": " corresponds to the request to be retried.\nSecond argument is the " + }, + { + "kind": "code", + "text": "`Error`" + }, + { + "kind": "text", + "text": " instance that\nrepresents the last error thrown during processing of the request." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 218, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L218" + } + ], + "type": { + "type": "reference", + "target": 7902, + "typeArguments": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 7972, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "ErrorHandler", + "package": "@crawlee/basic" + } + }, + { + "id": 7965, + "name": "experiments", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enables experimental features of Crawlee, which can alter the behavior of the crawler.\nWARNING: these options are not guaranteed to be stable and may change or be removed at any time." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 376, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L376" + } + ], + "type": { + "type": "reference", + "target": 7973, + "name": "CrawlerExperiments", + "package": "@crawlee/basic" + } + }, + { + "id": 7932, + "name": "extendContext", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows the user to extend the crawling context passed to the request handler with custom functionality.\n\n**Example usage:**\n\n" + }, + { + "kind": "code", + "text": "```javascript\nimport { BasicCrawler } from 'crawlee';\n\n// Create a crawler instance\nconst crawler = new BasicCrawler({\n extendContext(context) => ({\n async customHelper() {\n await context.pushData({ url: context.request.url })\n }\n }),\n async requestHandler(context) {\n await context.customHelper();\n },\n});\n```" + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 170, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L170" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7933, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 170, + "character": 20, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L170" + } + ], + "signatures": [ + { + "id": 7934, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 170, + "character": 20, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L170" + } + ], + "parameters": [ + { + "id": 7935, + "name": "context", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "reference", + "target": 7970, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.Context", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Awaitable" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7971, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.ContextExtension", + "refersToTypeParameter": true + } + ], + "name": "Awaitable", + "package": "@crawlee/types" + } + } + ] + } + } + }, + { + "id": 7944, + "name": "failedRequestHandler", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A function to handle requests that failed more than " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`" + }, + { + "kind": "text", + "text": " times.\n\nThe function receives the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext" + }, + { + "kind": "text", + "text": " as the first argument,\nwhere the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext.request|`request`" + }, + { + "kind": "text", + "text": " corresponds to the failed request.\nSecond argument is the " + }, + { + "kind": "code", + "text": "`Error`" + }, + { + "kind": "text", + "text": " instance that\nrepresents the last error thrown during processing of the request." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 228, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L228" + } + ], + "type": { + "type": "reference", + "target": 7902, + "typeArguments": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 7972, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "ErrorHandler", + "package": "@crawlee/basic" + } + }, + { + "id": 7967, + "name": "httpClient", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP client implementation for the " + }, + { + "kind": "code", + "text": "`sendRequest`" + }, + { + "kind": "text", + "text": " context helper and for plain HTTP crawling.\nDefaults to a new instance of " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "GotScrapingHttpClient" + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 388, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L388" + } + ], + "type": { + "type": "reference", + "target": 15897, + "name": "BaseHttpClient", + "package": "@crawlee/types" + } + }, + { + "id": 7969, + "name": "id", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier for the crawler instance. This ID is used to isolate the state returned by\n" + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.useState|`crawler.useState()`" + }, + { + "kind": "text", + "text": " from other crawler instances.\n\nWhen multiple crawler instances use " + }, + { + "kind": "code", + "text": "`useState()`" + }, + { + "kind": "text", + "text": " without an explicit " + }, + { + "kind": "code", + "text": "`id`" + }, + { + "kind": "text", + "text": ", they will share the same\nstate object for backward compatibility. A warning will be logged in this case.\n\nTo ensure each crawler has its own isolated state that also persists across script restarts\n(e.g., during Apify migrations), provide a stable, unique ID for each crawler instance." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 407, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L407" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7954, + "name": "keepAlive", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows to keep the crawler alive even if the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": " gets empty.\nBy default, the " + }, + { + "kind": "code", + "text": "`crawler.run()`" + }, + { + "kind": "text", + "text": " will resolve once the queue is empty. With " + }, + { + "kind": "code", + "text": "`keepAlive: true`" + }, + { + "kind": "text", + "text": " it will keep running,\nwaiting for more requests to come. Use " + }, + { + "kind": "code", + "text": "`crawler.stop()`" + }, + { + "kind": "text", + "text": " to exit the crawler gracefully, or " + }, + { + "kind": "code", + "text": "`crawler.teardown()`" + }, + { + "kind": "text", + "text": " to stop it immediately." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 305, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L305" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + }, + { + "id": 7952, + "name": "maxConcurrency", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the\nAutoscaledPool " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`" + }, + { + "kind": "text", + "text": " option." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 291, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L291" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7949, + "name": "maxCrawlDepth", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum depth of the crawl. If not set, the crawl will continue until all requests are processed.\nSetting this to " + }, + { + "kind": "code", + "text": "`0`" + }, + { + "kind": "text", + "text": " will only process the initial requests, skipping all links enqueued by " + }, + { + "kind": "code", + "text": "`crawlingContext.enqueueLinks`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "code", + "text": "`crawlingContext.addRequests`" + }, + { + "kind": "text", + "text": ".\nPassing " + }, + { + "kind": "code", + "text": "`1`" + }, + { + "kind": "text", + "text": " will process the initial requests and all links enqueued by " + }, + { + "kind": "code", + "text": "`crawlingContext.enqueueLinks`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "code", + "text": "`crawlingContext.addRequests`" + }, + { + "kind": "text", + "text": " in the handler for initial requests." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 268, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L268" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7945, + "name": "maxRequestRetries", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions\n(" + }, + { + "kind": "code", + "text": "`requestHandler`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`preNavigationHooks`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`postNavigationHooks`" + }, + { + "kind": "text", + "text": ").\n\nThis limit does not apply to retries triggered by session rotation\n(see " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`" + }, + { + "kind": "text", + "text": ")." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\n3\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 239, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L239" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7948, + "name": "maxRequestsPerCrawl", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.\nThis value should always be set in order to prevent infinite loops in misconfigured crawlers.\n> *NOTE:* In cases of parallel crawling, the actual number of pages visited might be slightly higher than this value." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 261, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L261" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7953, + "name": "maxRequestsPerMinute", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of requests per minute the crawler should run.\nBy default, this is set to " + }, + { + "kind": "code", + "text": "`Infinity`" + }, + { + "kind": "text", + "text": ", but we can pass any positive, non-zero integer.\nShortcut for the AutoscaledPool " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`" + }, + { + "kind": "text", + "text": " option." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 298, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L298" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7947, + "name": "maxSessionRotations", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of session rotations per request.\nThe crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.\n\nThe session rotations are not counted towards the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`" + }, + { + "kind": "text", + "text": " limit." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\n10\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 254, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L254" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7951, + "name": "minConcurrency", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the\nAutoscaledPool " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "AutoscaledPoolOptions.minConcurrency|`minConcurrency`" + }, + { + "kind": "text", + "text": " option.\n> *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.\nIf not sure, it's better to keep the default value and the concurrency will scale up automatically." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 285, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L285" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7963, + "name": "onSkippedRequest", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "When a request is skipped for some reason, you can use this callback to act on it.\nThis is currently fired for requests skipped\n1. based on robots.txt file,\n2. because they don't match enqueueLinks filters,\n3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,\n4. or because the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`" + }, + { + "kind": "text", + "text": " limit has been reached" + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 367, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L367" + } + ], + "type": { + "type": "reference", + "target": 1217, + "name": "SkippedRequestCallback", + "package": "@crawlee/core" + } + }, + { + "id": 7968, + "name": "proxyConfiguration", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set, the crawler will be configured for all connections to use\nthe Proxy URLs provided and rotated according to the configuration." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 394, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L394" + } + ], + "type": { + "type": "reference", + "target": 1926, + "name": "ProxyConfiguration", + "package": "@crawlee/core" + } + }, + { + "id": 7931, + "name": "requestHandler", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "User-provided function that performs the logic of the crawler. It is called for each URL to crawl.\n\nThe function receives the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext" + }, + { + "kind": "text", + "text": " as an argument,\nwhere the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlingContext.request|`request`" + }, + { + "kind": "text", + "text": " represents the URL to crawl.\n\nThe function must return a promise, which is then awaited by the crawler.\n\nIf the function throws an exception, the crawler will try to re-crawl the\nrequest later, up to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`" + }, + { + "kind": "text", + "text": " times.\nIf all the retries fail, the crawler calls the function\nprovided to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`" + }, + { + "kind": "text", + "text": " parameter.\nTo make this work, we should **always**\nlet our function throw exceptions rather than catch them.\nThe exceptions are logged to the request using the\n" + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request.pushErrorMessage|`Request.pushErrorMessage()`" + }, + { + "kind": "text", + "text": " function." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 147, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L147" + } + ], + "type": { + "type": "reference", + "target": 7897, + "typeArguments": [ + { + "type": "reference", + "target": 7972, + "name": "ExtendedContext", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.ExtendedContext", + "refersToTypeParameter": true + } + ], + "name": "RequestHandler", + "package": "@crawlee/basic" + } + }, + { + "id": 7942, + "name": "requestHandlerTimeoutSecs", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout in which the function passed as " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestHandler|`requestHandler`" + }, + { + "kind": "text", + "text": " needs to finish, in seconds." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\n60\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 207, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L207" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7939, + "name": "requestList", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Static list of URLs to be processed.\nIf not provided, the crawler will open the default request queue when the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.addRequests|`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " function is called.\n> Alternatively, " + }, + { + "kind": "code", + "text": "`requests`" + }, + { + "kind": "text", + "text": " parameter of " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.run|`crawler.run()`" + }, + { + "kind": "text", + "text": " could be used to enqueue the initial requests -\nit is a shortcut for running " + }, + { + "kind": "code", + "text": "`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " before the " + }, + { + "kind": "code", + "text": "`crawler.run()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 185, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L185" + } + ], + "type": { + "type": "reference", + "target": 2992, + "name": "IRequestList", + "package": "@crawlee/core" + } + }, + { + "id": 7941, + "name": "requestManager", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows explicitly configuring a request manager. Mutually exclusive with the " + }, + { + "kind": "code", + "text": "`requestQueue`" + }, + { + "kind": "text", + "text": " and " + }, + { + "kind": "code", + "text": "`requestList`" + }, + { + "kind": "text", + "text": " options.\n\nThis enables explicitly configuring the crawler to use " + }, + { + "kind": "code", + "text": "`RequestManagerTandem`" + }, + { + "kind": "text", + "text": ", for instance.\nIf using this, the type of " + }, + { + "kind": "code", + "text": "`BasicCrawler.requestQueue`" + }, + { + "kind": "text", + "text": " may not be fully compatible with the " + }, + { + "kind": "code", + "text": "`RequestProvider`" + }, + { + "kind": "text", + "text": " class." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 201, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L201" + } + ], + "type": { + "type": "reference", + "target": 3161, + "name": "IRequestManager", + "package": "@crawlee/core" + } + }, + { + "id": 7940, + "name": "requestQueue", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.\nIf not provided, the crawler will open the default request queue when the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.addRequests|`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " function is called.\n> Alternatively, " + }, + { + "kind": "code", + "text": "`requests`" + }, + { + "kind": "text", + "text": " parameter of " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawler.run|`crawler.run()`" + }, + { + "kind": "text", + "text": " could be used to enqueue the initial requests -\nit is a shortcut for running " + }, + { + "kind": "code", + "text": "`crawler.addRequests()`" + }, + { + "kind": "text", + "text": " before the " + }, + { + "kind": "code", + "text": "`crawler.run()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 193, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L193" + } + ], + "type": { + "type": "reference", + "target": 3192, + "name": "RequestProvider", + "package": "@crawlee/core" + } + }, + { + "id": 7960, + "name": "respectRobotsTxtFile", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": ", the crawler will automatically try to fetch the robots.txt file for each domain,\nand skip those that are not allowed. This also prevents disallowed URLs to be added via " + }, + { + "kind": "code", + "text": "`enqueueLinks`" + }, + { + "kind": "text", + "text": ".\n\nIf an object is provided, it may contain a " + }, + { + "kind": "code", + "text": "`userAgent`" + }, + { + "kind": "text", + "text": " property to specify which user-agent\nshould be used when checking the robots.txt file. If not provided, the default user-agent " + }, + { + "kind": "code", + "text": "`*`" + }, + { + "kind": "text", + "text": " will be used." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 357, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L357" + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "boolean" + }, + { + "type": "reflection", + "declaration": { + "id": 7961, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "children": [ + { + "id": 7962, + "name": "userAgent", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 357, + "character": 39, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L357" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7962 + ] + } + ], + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 357, + "character": 37, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L357" + } + ] + } + } + ] + } + }, + { + "id": 7959, + "name": "retryOnBlocked", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": ", the crawler will automatically try to bypass any detected bot protection.\n\nCurrently supports:\n- [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)\n- [**Google Search** Rate Limiting](https://www.google.com/sorry/)" + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 348, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L348" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + }, + { + "id": 7946, + "name": "sameDomainDelaySecs", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates how much time (in seconds) to wait before crawling another same domain request." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\n0\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 245, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L245" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7956, + "name": "sessionPoolOptions", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration options for " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "SessionPool" + }, + { + "kind": "text", + "text": " to use." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 316, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L316" + } + ], + "type": { + "type": "reference", + "target": 2284, + "name": "SessionPoolOptions", + "package": "@crawlee/core" + } + }, + { + "id": 7966, + "name": "statisticsOptions", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Customize the way statistics collecting works, such as logging interval or\nwhether to output them to the Key-Value store." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 382, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L382" + } + ], + "type": { + "type": "reference", + "target": 971, + "name": "StatisticsOptions", + "package": "@crawlee/core" + } + }, + { + "id": 7958, + "name": "statusMessageCallback", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows overriding the default status message. The callback needs to call " + }, + { + "kind": "code", + "text": "`crawler.setStatusMessage()`" + }, + { + "kind": "text", + "text": " explicitly.\nThe default status message is provided in the parameters.\n\n" + }, + { + "kind": "code", + "text": "```ts\nconst crawler = new CheerioCrawler({\n statusMessageCallback: async (ctx) => {\n return ctx.crawler.setStatusMessage(`this is status message from ${new Date().toISOString()}`, { level: 'INFO' }); // log level defaults to 'DEBUG'\n },\n statusMessageLoggingInterval: 1, // defaults to 10s\n async requestHandler({ $, enqueueLinks, request, log }) {\n // ...\n },\n});\n```" + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 339, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L339" + } + ], + "type": { + "type": "reference", + "target": 7916, + "typeArguments": [ + { + "type": "reference", + "target": 7836, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "BasicCrawlingContext", + "package": "@crawlee/basic" + }, + { + "type": "reference", + "target": 7975, + "typeArguments": [ + { + "type": "reference", + "target": 7836, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "BasicCrawlingContext", + "package": "@crawlee/basic" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "never" + } + ], + "name": "Dictionary", + "package": "@crawlee/types" + }, + { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": 7836, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "BasicCrawlingContext", + "package": "@crawlee/basic" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "never" + } + ], + "name": "Dictionary", + "package": "@crawlee/types" + } + ] + } + ], + "name": "BasicCrawler", + "package": "@crawlee/basic" + } + ], + "name": "StatusMessageCallback", + "package": "@crawlee/basic" + } + }, + { + "id": 7957, + "name": "statusMessageLoggingInterval", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Defines the length of the interval for calling the " + }, + { + "kind": "code", + "text": "`setStatusMessage`" + }, + { + "kind": "text", + "text": " in seconds." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 321, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L321" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7955, + "name": "useSessionPool", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Basic crawler will initialize the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "SessionPool" + }, + { + "kind": "text", + "text": " with the corresponding " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "SessionPoolOptions|`sessionPoolOptions`" + }, + { + "kind": "text", + "text": ".\nThe session instance will be than available in the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BasicCrawlerOptions.requestHandler|`requestHandler`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 311, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L311" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7950, + 7936, + 7943, + 7965, + 7932, + 7944, + 7967, + 7969, + 7954, + 7952, + 7949, + 7945, + 7948, + 7953, + 7947, + 7951, + 7963, + 7968, + 7931, + 7942, + 7939, + 7941, + 7940, + 7960, + 7959, + 7946, + 7956, + 7966, + 7958, + 7957, + 7955 + ] + } + ], + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 125, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L125" + } + ], + "typeParameters": [ + { + "id": 7970, + "name": "Context", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 753, + "name": "CrawlingContext", + "package": "@crawlee/core" + }, + "default": { + "type": "reference", + "target": 753, + "name": "CrawlingContext", + "package": "@crawlee/core" + } + }, + { + "id": 7971, + "name": "ContextExtension", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "never" + } + ], + "name": "Dictionary", + "package": "@crawlee/types" + } + }, + { + "id": 7972, + "name": "ExtendedContext", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": 7970, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.Context", + "refersToTypeParameter": true + }, + "default": { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": 7970, + "name": "Context", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.Context", + "refersToTypeParameter": true + }, + { + "type": "reference", + "target": 7971, + "name": "ContextExtension", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlerOptions.ContextExtension", + "refersToTypeParameter": true + } + ] + } + } + ], + "extendedBy": [ + { + "type": "reference", + "target": 9177, + "name": "HttpCrawlerOptions" + } + ] + }, + { + "id": 7836, + "name": "BasicCrawlingContext", + "variant": "declaration", + "kind": 256, + "flags": {}, + "children": [ + { + "id": 7878, + "name": "addRequests", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests directly to the request queue." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 86, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L86" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7879, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 86, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L86" + } + ], + "signatures": [ + { + "id": 7880, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 86, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L86" + } + ], + "parameters": [ + { + "id": 7881, + "name": "requestsLike", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyObjectDeep" + }, + "typeArguments": [ + { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Partial" + }, + "typeArguments": [ + { + "type": "reference", + "target": 1970, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "RequestOptions", + "package": "@crawlee/core" + } + ], + "name": "Partial", + "package": "typescript" + }, + { + "type": "reflection", + "declaration": { + "id": 7882, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "children": [ + { + "id": 7884, + "name": "regex", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 577, + "character": 76, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L577" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "RegExp" + }, + "name": "RegExp", + "package": "typescript" + } + }, + { + "id": 7883, + "name": "requestsFromUrl", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 577, + "character": 50, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L577" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7884, + 7883 + ] + } + ], + "sources": [ + { + "fileName": "packages/core/src/request.ts", + "line": 577, + "character": 48, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/request.ts#L577" + } + ] + } + } + ] + } + ], + "name": "ReadonlyObjectDeep", + "package": "type-fest" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyObjectDeep" + }, + "typeArguments": [ + { + "type": "reference", + "target": 1999, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + ], + "name": "CrawleeRequest", + "package": "@crawlee/core" + } + ], + "name": "ReadonlyObjectDeep", + "package": "type-fest" + } + ] + } + } + } + }, + { + "id": 7885, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Options for the request queue" + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyObjectDeep" + }, + "typeArguments": [ + { + "type": "reference", + "target": 3310, + "name": "RequestQueueOperationOptions", + "package": "@crawlee/core" + } + ], + "name": "ReadonlyObjectDeep", + "package": "type-fest" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "void" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + }, + "inheritedFrom": { + "type": "reference", + "target": 795, + "name": "CrawlingContext.addRequests" + } + }, + { + "id": 7891, + "name": "getKeyValueStore", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a key-value store with given name or id, or the default one for the crawler." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 99, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L99" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7892, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 99, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L99" + } + ], + "signatures": [ + { + "id": 7893, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 99, + "character": 22, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L99" + } + ], + "parameters": [ + { + "id": 7894, + "name": "idOrName", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Pick" + }, + "typeArguments": [ + { + "type": "reference", + "target": 2889, + "name": "KeyValueStore", + "package": "@crawlee/core" + }, + { + "type": "union", + "types": [ + { + "type": "literal", + "value": "id" + }, + { + "type": "literal", + "value": "name" + }, + { + "type": "literal", + "value": "getValue" + }, + { + "type": "literal", + "value": "getAutoSavedValue" + }, + { + "type": "literal", + "value": "setValue" + }, + { + "type": "literal", + "value": "getPublicUrl" + } + ] + } + ], + "name": "Pick", + "package": "typescript" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + }, + "inheritedFrom": { + "type": "reference", + "target": 808, + "name": "CrawlingContext.getKeyValueStore" + } + }, + { + "id": 7870, + "name": "id", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 29, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L29" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + }, + "inheritedFrom": { + "type": "reference", + "target": 787, + "name": "CrawlingContext.id" + } + }, + { + "id": 7895, + "name": "log", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A preconfigured logger for the request handler." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 106, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L106" + } + ], + "type": { + "type": "reference", + "target": 1349, + "name": "Log", + "package": "@apify/log" + }, + "inheritedFrom": { + "type": "reference", + "target": 812, + "name": "CrawlingContext.log" + } + }, + { + "id": 7872, + "name": "proxyInfo", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An object with information about currently used proxy by the crawler\nand configured by the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "ProxyConfiguration" + }, + { + "kind": "text", + "text": " class." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 36, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L36" + } + ], + "type": { + "type": "reference", + "target": 15902, + "name": "ProxyInfo", + "package": "@crawlee/types" + }, + "inheritedFrom": { + "type": "reference", + "target": 789, + "name": "CrawlingContext.proxyInfo" + } + }, + { + "id": 7873, + "name": "request", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The original " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " object." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 41, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L41" + } + ], + "type": { + "type": "reference", + "target": 1999, + "typeArguments": [ + { + "type": "reference", + "target": 7896, + "name": "UserData", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlingContext.UserData", + "refersToTypeParameter": true + } + ], + "name": "CrawleeRequest", + "package": "@crawlee/core" + }, + "inheritedFrom": { + "type": "reference", + "target": 790, + "name": "CrawlingContext.request" + } + }, + { + "id": 7860, + "name": "sendRequest", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.\n\nThis is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).\nCheck the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for\nmore detailed explanation of how to do that.\n\n" + }, + { + "kind": "code", + "text": "```ts\nasync requestHandler({ sendRequest }) {\n const { body } = await sendRequest({\n // override headers only\n headers: { ... },\n });\n},\n```" + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 156, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L156" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7861, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 156, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L156" + } + ], + "signatures": [ + { + "id": 7862, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 156, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L156" + } + ], + "parameters": [ + { + "id": 7863, + "name": "requestOverrides", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Partial" + }, + "typeArguments": [ + { + "type": "reference", + "target": 15847, + "name": "HttpRequestOptions", + "package": "@crawlee/types" + } + ], + "name": "Partial", + "package": "typescript" + } + }, + { + "id": 7864, + "name": "optionsOverrides", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "reference", + "target": 15886, + "name": "SendRequestOptions", + "package": "@crawlee/types" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.dom.d.ts", + "qualifiedName": "Response" + }, + "name": "Response", + "package": "typescript" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + }, + "inheritedFrom": { + "type": "reference", + "target": 777, + "name": "CrawlingContext.sendRequest" + } + }, + { + "id": 7871, + "name": "session", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 30, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L30" + } + ], + "type": { + "type": "reference", + "target": 2200, + "name": "Session", + "package": "@crawlee/core" + }, + "inheritedFrom": { + "type": "reference", + "target": 788, + "name": "CrawlingContext.session" + } + }, + { + "id": 7886, + "name": "useState", + "variant": "declaration", + "kind": 1024, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Returns the state - a piece of mutable persistent data shared across all the request handler runs." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 94, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L94" + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 7887, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 94, + "character": 14, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L94" + } + ], + "signatures": [ + { + "id": 7888, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 94, + "character": 14, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L94" + } + ], + "typeParameters": [ + { + "id": 7889, + "name": "State", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + }, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + } + ], + "parameters": [ + { + "id": 7890, + "name": "defaultValue", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "reference", + "target": 745, + "name": "State", + "package": "@crawlee/core", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "reference", + "target": 745, + "name": "State", + "package": "@crawlee/core", + "refersToTypeParameter": true + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + }, + "inheritedFrom": { + "type": "reference", + "target": 803, + "name": "CrawlingContext.useState" + } + }, + { + "id": 7837, + "name": "enqueueLinks", + "variant": "declaration", + "kind": 2048, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "This function automatically finds and enqueues links from the current page, adding them to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": "\ncurrently used by the crawler.\n\nOptionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions\nand override settings of the enqueued " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects.\n\nCheck out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example\nfor more details regarding its usage.\n\n**Example usage**\n\n" + }, + { + "kind": "code", + "text": "```ts\nasync requestHandler({ enqueueLinks }) {\n await enqueueLinks({\n globs: [\n 'https://www.example.com/handbags/*',\n ],\n });\n},\n```" + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 135, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L135" + } + ], + "signatures": [ + { + "id": 7838, + "name": "enqueueLinks", + "variant": "signature", + "kind": 4096, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "This function automatically finds and enqueues links from the current page, adding them to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": "\ncurrently used by the crawler.\n\nOptionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions\nand override settings of the enqueued " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects.\n\nCheck out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example\nfor more details regarding its usage.\n\n**Example usage**\n\n" + }, + { + "kind": "code", + "text": "```ts\nasync requestHandler({ enqueueLinks }) {\n await enqueueLinks({\n globs: [\n 'https://www.example.com/handbags/*',\n ],\n });\n},\n```" + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "Promise that resolves to " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "BatchAddRequestsResult" + }, + { + "kind": "text", + "text": " object." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 135, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L135" + } + ], + "parameters": [ + { + "id": 7839, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "All " + }, + { + "kind": "code", + "text": "`enqueueLinks()`" + }, + { + "kind": "text", + "text": " parameters are passed via an options object." + } + ] + }, + "type": { + "type": "intersection", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyObjectDeep" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Omit" + }, + "typeArguments": [ + { + "type": "reflection", + "declaration": { + "id": 7840, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "children": [ + { + "id": 7847, + "name": "baseUrl", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,\nsince the relative URL resolution is done inside the browser automatically." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 68, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L68" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7849, + "name": "exclude", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of glob pattern strings, regexp patterns or plain objects\ncontaining patterns matching URLs that will **never** be enqueued.\n\nThe plain objects must include either the " + }, + { + "kind": "code", + "text": "`glob`" + }, + { + "kind": "text", + "text": " property or the " + }, + { + "kind": "code", + "text": "`regexp`" + }, + { + "kind": "text", + "text": " property.\n\nGlob matching is always case-insensitive.\nIf you need case-sensitive matching, provide a regexp." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 94, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L94" + } + ], + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "union", + "types": [ + { + "type": "reference", + "target": 1211, + "name": "GlobInput", + "package": "@crawlee/core" + }, + { + "type": "reference", + "target": 1215, + "name": "RegExpInput", + "package": "@crawlee/core" + } + ] + } + } + } + }, + { + "id": 7857, + "name": "forefront", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": ":\n - while adding the request to the queue: the request will be added to the foremost position in the queue.\n - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned\n in the next call to " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue.fetchNextRequest" + }, + { + "kind": "text", + "text": ".\nBy default, it's put to the end of the queue.\n\nIn case the request is already present in the queue, this option has no effect.\n\nIf more requests are added with this option at once, their order in the following " + }, + { + "kind": "code", + "text": "`fetchNextRequest`" + }, + { + "kind": "text", + "text": " call\nis arbitrary." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\nfalse\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/storages/request_provider.ts", + "line": 959, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/storages/request_provider.ts#L959" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + }, + { + "id": 7848, + "name": "globs", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of glob pattern strings or plain objects\ncontaining glob pattern strings matching the URLs to be enqueued.\n\nThe plain objects must include at least the " + }, + { + "kind": "code", + "text": "`glob`" + }, + { + "kind": "text", + "text": " property, which holds the glob pattern string.\nAll remaining keys will be used as request options for the corresponding enqueued " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects.\n\nThe matching is always case-insensitive.\nIf you need case-sensitive matching, use " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " property directly.\n\nIf " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": " is an empty array or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": ", and " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " are also not defined, then the function\nenqueues the links with the same subdomain." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 83, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L83" + } + ], + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "reference", + "target": 1211, + "name": "GlobInput", + "package": "@crawlee/core" + } + } + } + }, + { + "id": 7845, + "name": "label", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sets " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request.label" + }, + { + "kind": "text", + "text": " for newly enqueued requests.\n\nNote that the request options specified in " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": ", or " + }, + { + "kind": "code", + "text": "`pseudoUrls`" + }, + { + "kind": "text", + "text": " objects\nhave priority over this option." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 56, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L56" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7841, + "name": "limit", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 36, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L36" + } + ], + "type": { + "type": "intrinsic", + "name": "number" + } + }, + { + "id": 7856, + "name": "onSkippedRequest", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "When a request is skipped for some reason, you can use this callback to act on it.\nThis is currently fired for requests skipped\n1. based on robots.txt file,\n2. because they don't match enqueueLinks filters,\n3. or because the maxRequestsPerCrawl limit has been reached" + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 192, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L192" + } + ], + "type": { + "type": "reference", + "target": 1217, + "name": "SkippedRequestCallback", + "package": "@crawlee/core" + } + }, + { + "id": 7851, + "name": "pseudoUrls", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "*NOTE:* In future versions of SDK the options will be removed.\nPlease use " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " instead.\n\nAn array of " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "PseudoUrl" + }, + { + "kind": "text", + "text": " strings or plain objects\ncontaining " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "PseudoUrl" + }, + { + "kind": "text", + "text": " strings matching the URLs to be enqueued.\n\nThe plain objects must include at least the " + }, + { + "kind": "code", + "text": "`purl`" + }, + { + "kind": "text", + "text": " property, which holds the pseudo-URL string.\nAll remaining keys will be used as request options for the corresponding enqueued " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects.\n\nWith a pseudo-URL string, the matching is always case-insensitive.\nIf you need case-sensitive matching, use " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " property directly.\n\nIf " + }, + { + "kind": "code", + "text": "`pseudoUrls`" + }, + { + "kind": "text", + "text": " is an empty array or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": ", then the function\nenqueues the links with the same subdomain." + } + ], + "blockTags": [ + { + "tag": "@deprecated", + "content": [ + { + "kind": "text", + "text": "prefer using " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": " or " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " instead" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 126, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L126" + } + ], + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "reference", + "target": 1207, + "name": "PseudoUrlInput", + "package": "@crawlee/core" + } + } + } + }, + { + "id": 7850, + "name": "regexps", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of regular expressions or plain objects\ncontaining regular expressions matching the URLs to be enqueued.\n\nThe plain objects must include at least the " + }, + { + "kind": "code", + "text": "`regexp`" + }, + { + "kind": "text", + "text": " property, which holds the regular expression.\nAll remaining keys will be used as request options for the corresponding enqueued " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " objects.\n\nIf " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": " is an empty array or " + }, + { + "kind": "code", + "text": "`undefined`" + }, + { + "kind": "text", + "text": ", and " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": " are also not defined, then the function\nenqueues the links with the same subdomain." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 106, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L106" + } + ], + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "reference", + "target": 1215, + "name": "RegExpInput", + "package": "@crawlee/core" + } + } + } + }, + { + "id": 7842, + "name": "requestQueue", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A request queue to which the URLs will be enqueued." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 42, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L42" + } + ], + "type": { + "type": "reference", + "target": 3192, + "name": "RequestProvider", + "package": "@crawlee/core" + } + }, + { + "id": 7855, + "name": "robotsTxtFile", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "RobotsTxtFile instance for the current request that triggered the " + }, + { + "kind": "code", + "text": "`enqueueLinks`" + }, + { + "kind": "text", + "text": ".\nIf provided, disallowed URLs will be ignored." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 183, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L183" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Pick" + }, + "typeArguments": [ + { + "type": "reference", + "target": 15395, + "name": "RobotsTxtFile", + "package": "@crawlee/utils" + }, + { + "type": "literal", + "value": "isAllowed" + } + ], + "name": "Pick", + "package": "typescript" + } + }, + { + "id": 7843, + "name": "selector", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A CSS selector matching links to be enqueued." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 45, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L45" + } + ], + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7846, + "name": "skipNavigation", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to " + }, + { + "kind": "code", + "text": "`true`" + }, + { + "kind": "text", + "text": ", tells the crawler to skip navigation and process the request directly." + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\nfalse\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 62, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L62" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + }, + { + "id": 7853, + "name": "strategy", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy to use when enqueueing the urls.\n\nDepending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:\n\n" + }, + { + "kind": "code", + "text": "```md\nProtocol Domain\n┌────┐ ┌─────────┐\nhttps://example.crawlee.dev/...\n│ └─────────────────┤\n│ Hostname │\n│ │\n└─────────────────────────┘\n Origin\n```" + } + ], + "blockTags": [ + { + "tag": "@default", + "content": [ + { + "kind": "code", + "text": "```ts\nEnqueueStrategy.SameHostname\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 171, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L171" + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": 1148, + "name": "EnqueueStrategy", + "package": "@crawlee/core" + }, + { + "type": "literal", + "value": "all" + }, + { + "type": "literal", + "value": "same-domain" + }, + { + "type": "literal", + "value": "same-hostname" + }, + { + "type": "literal", + "value": "same-origin" + } + ] + } + }, + { + "id": 7852, + "name": "transformRequestFunction", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Just before a new " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request" + }, + { + "kind": "text", + "text": " is constructed and enqueued to the " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "RequestQueue" + }, + { + "kind": "text", + "text": ", this function can be used\nto remove it or modify its contents such as " + }, + { + "kind": "code", + "text": "`userData`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`payload`" + }, + { + "kind": "text", + "text": " or, most importantly " + }, + { + "kind": "code", + "text": "`uniqueKey`" + }, + { + "kind": "text", + "text": ". This is useful\nwhen you need to enqueue multiple " + }, + { + "kind": "code", + "text": "`Requests`" + }, + { + "kind": "text", + "text": " to the queue that share the same URL, but differ in methods or payloads,\nor to dynamically update or create " + }, + { + "kind": "code", + "text": "`userData`" + }, + { + "kind": "text", + "text": ".\n\nFor example: by adding " + }, + { + "kind": "code", + "text": "`keepUrlFragment: true`" + }, + { + "kind": "text", + "text": " to the " + }, + { + "kind": "code", + "text": "`request`" + }, + { + "kind": "text", + "text": " object, URL fragments will not be removed\nwhen " + }, + { + "kind": "code", + "text": "`uniqueKey`" + }, + { + "kind": "text", + "text": " is computed.\n\n**Example:**\n" + }, + { + "kind": "code", + "text": "```javascript\n{\n transformRequestFunction: (request) => {\n request.userData.foo = 'bar';\n request.keepUrlFragment = true;\n return request;\n }\n}\n```" + }, + { + "kind": "text", + "text": "\n\nNote that the request options specified in " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": ", or " + }, + { + "kind": "code", + "text": "`pseudoUrls`" + }, + { + "kind": "text", + "text": " objects\nhave priority over this function. Some request options returned by " + }, + { + "kind": "code", + "text": "`transformRequestFunction`" + }, + { + "kind": "text", + "text": " may be overwritten by pattern-based options from " + }, + { + "kind": "code", + "text": "`globs`" + }, + { + "kind": "text", + "text": ", " + }, + { + "kind": "code", + "text": "`regexps`" + }, + { + "kind": "text", + "text": ", or " + }, + { + "kind": "code", + "text": "`pseudoUrls`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 151, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L151" + } + ], + "type": { + "type": "reference", + "target": 1224, + "name": "RequestTransform", + "package": "@crawlee/core" + } + }, + { + "id": 7859, + "name": "urls", + "variant": "declaration", + "kind": 1024, + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of URLs to enqueue." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 39, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L39" + } + ], + "type": { + "type": "typeOperator", + "operator": "readonly", + "target": { + "type": "array", + "elementType": { + "type": "intrinsic", + "name": "string" + } + } + } + }, + { + "id": 7844, + "name": "userData", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sets " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Request.userData" + }, + { + "kind": "text", + "text": " for newly enqueued requests." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 48, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L48" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + }, + { + "id": 7854, + "name": "waitForAllRequestsToBeAdded", + "variant": "declaration", + "kind": 1024, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.\nYou can use this option to wait for adding all of them." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/enqueue_links/enqueue_links.ts", + "line": 177, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/enqueue_links/enqueue_links.ts#L177" + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7847, + 7849, + 7857, + 7848, + 7845, + 7841, + 7856, + 7851, + 7850, + 7842, + 7855, + 7843, + 7846, + 7853, + 7852, + 7859, + 7844, + 7854 + ] + } + ], + "sources": [ + { + "fileName": "node_modules/type-fest/source/simplify.d.ts", + "line": 58, + "character": 26 + } + ] + } + }, + { + "type": "union", + "types": [ + { + "type": "literal", + "value": "requestQueue" + }, + { + "type": "literal", + "value": "robotsTxtFile" + } + ] + } + ], + "name": "Omit", + "package": "typescript" + } + ], + "name": "ReadonlyObjectDeep", + "package": "type-fest" + }, + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Pick" + }, + "typeArguments": [ + { + "type": "reference", + "target": 1128, + "name": "EnqueueLinksOptions", + "package": "@crawlee/core" + }, + { + "type": "union", + "types": [ + { + "type": "literal", + "value": "requestQueue" + }, + { + "type": "literal", + "value": "robotsTxtFile" + } + ] + } + ], + "name": "Pick", + "package": "typescript" + } + ] + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "unknown" + } + ], + "name": "Promise", + "package": "typescript" + }, + "inheritedFrom": { + "type": "reference", + "target": 755, + "name": "CrawlingContext.enqueueLinks" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": 754, + "name": "CrawlingContext.enqueueLinks" + } + }, + { + "id": 7874, + "name": "pushData", + "variant": "declaration", + "kind": 2048, + "flags": { + "isInherited": true + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 50, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L50" + } + ], + "signatures": [ + { + "id": 7875, + "name": "pushData", + "variant": "signature", + "kind": 4096, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "This function allows you to push data to a " + }, + { + "kind": "inline-tag", + "tag": "@apilink", + "text": "Dataset" + }, + { + "kind": "text", + "text": " specified by name, or the one currently used by the crawler.\n\nShortcut for " + }, + { + "kind": "code", + "text": "`crawler.pushData()`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 50, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L50" + } + ], + "parameters": [ + { + "id": 7876, + "name": "data", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data to be pushed to the default dataset." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/type-fest/source/readonly-deep.d.ts", + "qualifiedName": "ReadonlyDeep" + }, + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + } + ] + } + ], + "name": "ReadonlyDeep", + "package": "type-fest" + } + }, + { + "id": 7877, + "name": "datasetIdOrName", + "variant": "param", + "kind": 32768, + "flags": { + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "string" + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "void" + } + ], + "name": "Promise", + "package": "typescript" + }, + "inheritedFrom": { + "type": "reference", + "target": 792, + "name": "CrawlingContext.pushData" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": 791, + "name": "CrawlingContext.pushData" + } + }, + { + "id": 7865, + "name": "registerDeferredCleanup", + "variant": "declaration", + "kind": 2048, + "flags": { + "isInherited": true + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 164, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L164" + } + ], + "signatures": [ + { + "id": 7866, + "name": "registerDeferredCleanup", + "variant": "signature", + "kind": 4096, + "flags": { + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance." + } + ] + }, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 164, + "character": 4, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L164" + } + ], + "parameters": [ + { + "id": 7867, + "name": "cleanup", + "variant": "param", + "kind": 32768, + "flags": {}, + "type": { + "type": "reflection", + "declaration": { + "id": 7868, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 164, + "character": 37, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L164" + } + ], + "signatures": [ + { + "id": 7869, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": {}, + "sources": [ + { + "fileName": "packages/core/src/crawlers/crawler_commons.ts", + "line": 164, + "character": 37, + "url": "https://github.com/apify/crawlee/blob/master/packages/core/src/crawlers/crawler_commons.ts#L164" + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "Promise" + }, + "typeArguments": [ + { + "type": "intrinsic", + "name": "unknown" + } + ], + "name": "Promise", + "package": "typescript" + } + } + ] + } + } + } + ], + "type": { + "type": "intrinsic", + "name": "void" + }, + "inheritedFrom": { + "type": "reference", + "target": 783, + "name": "CrawlingContext.registerDeferredCleanup" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": 782, + "name": "CrawlingContext.registerDeferredCleanup" + } + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 7878, + 7891, + 7870, + 7895, + 7872, + 7873, + 7860, + 7871, + 7886 + ] + }, + { + "title": "Methods", + "children": [ + 7837, + 7874, + 7865 + ] + } + ], + "sources": [ + { + "fileName": "packages/basic-crawler/src/internals/basic-crawler.ts", + "line": 83, + "character": 17, + "url": "https://github.com/apify/crawlee/blob/master/packages/basic-crawler/src/internals/basic-crawler.ts#L83" + } + ], + "typeParameters": [ + { + "id": 7896, + "name": "UserData", + "variant": "typeParam", + "kind": 131072, + "flags": {}, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + }, + "default": { + "type": "reference", + "target": { + "sourceFileName": "../packages/types/src/utility-types.ts", + "qualifiedName": "Dictionary" + }, + "name": "Dictionary", + "package": "@crawlee/types" + } + } + ], + "extendedTypes": [ + { + "type": "reference", + "target": 753, + "typeArguments": [ + { + "type": "reference", + "target": 7896, + "name": "UserData", + "package": "@crawlee/basic", + "qualifiedName": "BasicCrawlingContext.UserData", + "refersToTypeParameter": true + } + ], + "name": "CrawlingContext", + "package": "@crawlee/core" + } + ] + }, + { + "id": 6980, + "name": "CheerioAPI", + "variant": "declaration", + "kind": 256, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "A querying function, bound to a document created from the provided markup.\n\nAlso provides several helper methods for dealing with the document as a\nwhole." + } + ] + }, + "children": [ + { + "id": 6983, + "name": "fn", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mimic jQuery's prototype alias for plugin authors." + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/load.d.ts", + "line": 73, + "character": 4 + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "intrinsic", + "name": "any" + } + ], + "name": "Cheerio", + "package": "cheerio" + } + }, + { + "id": 6984, + "name": "load", + "variant": "declaration", + "kind": 1024, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "The " + }, + { + "kind": "code", + "text": "`.load`" + }, + { + "kind": "text", + "text": " static method defined on the \"loaded\" Cheerio factory function\nis deprecated. Users are encouraged to instead use the " + }, + { + "kind": "code", + "text": "`load`" + }, + { + "kind": "text", + "text": " function\nexported by the Cheerio module." + } + ], + "blockTags": [ + { + "tag": "@deprecated", + "content": [ + { + "kind": "text", + "text": "Use the " + }, + { + "kind": "code", + "text": "`load`" + }, + { + "kind": "text", + "text": " function exported by the Cheerio module." + } + ] + }, + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\nconst $ = cheerio.load('

      Hello, world.

      ');\n```" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/load.d.ts", + "line": 87, + "character": 4 + } + ], + "type": { + "type": "reflection", + "declaration": { + "id": 6985, + "name": "__type", + "variant": "declaration", + "kind": 65536, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/load.d.ts", + "line": 89, + "character": 154 + } + ], + "signatures": [ + { + "id": 6986, + "name": "__type", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/load.d.ts", + "line": 89, + "character": 154 + } + ], + "parameters": [ + { + "id": 6987, + "name": "content", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "intrinsic", + "name": "string" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/@types/node/buffer.buffer.d.ts", + "qualifiedName": "__global.Buffer" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayBufferLike" + }, + "name": "ArrayBufferLike", + "package": "typescript" + } + ], + "name": "Buffer", + "package": "@types/node", + "qualifiedName": "__global.Buffer" + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + }, + { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ] + } + }, + { + "id": 6988, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/options.ts", + "qualifiedName": "CheerioOptions" + }, + "name": "CheerioOptions", + "package": "cheerio" + } + ] + } + }, + { + "id": 6989, + "name": "isDocument", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + } + ] + } + } + }, + { + "id": 7018, + "name": "contains", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 86, + "character": 24 + } + ], + "signatures": [ + { + "id": 7019, + "name": "contains", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Checks to see if the " + }, + { + "kind": "code", + "text": "`contained`" + }, + { + "kind": "text", + "text": " DOM element is a descendant of the\n" + }, + { + "kind": "code", + "text": "`container`" + }, + { + "kind": "text", + "text": " DOM element." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "Indicates if the nodes contain one another." + } + ] + }, + { + "tag": "@alias", + "content": [ + { + "kind": "text", + "text": "Cheerio.contains" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/jQuery.contains/", + "target": "https://api.jquery.com/jQuery.contains/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 86, + "character": 24 + } + ], + "parameters": [ + { + "id": 7020, + "name": "container", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Potential parent node." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + { + "id": 7021, + "name": "contained", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Potential child node." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + } + ], + "type": { + "type": "intrinsic", + "name": "boolean" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.contains" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.contains" + } + }, + { + "id": 7022, + "name": "extract", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 96, + "character": 24 + } + ], + "signatures": [ + { + "id": 7023, + "name": "extract", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extract multiple values from a document, and store them in an object." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "An object containing the extracted values." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 96, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7024, + "name": "M", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/api/extract.ts", + "qualifiedName": "ExtractMap" + }, + "name": "ExtractMap", + "package": "cheerio" + } + } + ], + "parameters": [ + { + "id": 7025, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + }, + { + "id": 7026, + "name": "map", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "An object containing key-value pairs. The keys are the names of\n the properties to be created on the object, and the values are the\n selectors to be used to extract the values." + } + ] + }, + "type": { + "type": "reference", + "target": 7024, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + } + ], + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/api/extract.ts", + "qualifiedName": "ExtractedMap" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7024, + "name": "M", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ExtractedMap", + "package": "cheerio" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.extract" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.extract" + } + }, + { + "id": 6990, + "name": "html", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 14, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 23, + "character": 24 + } + ], + "signatures": [ + { + "id": 6991, + "name": "html", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Renders the document." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The rendered document." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 14, + "character": 24 + } + ], + "parameters": [ + { + "id": 6992, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + }, + { + "id": 6993, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Options for the renderer." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/options.ts", + "qualifiedName": "CheerioOptions" + }, + "name": "CheerioOptions", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.html" + } + }, + { + "id": 6994, + "name": "html", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Renders the document." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The rendered document." + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 23, + "character": 24 + } + ], + "parameters": [ + { + "id": 6995, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + }, + { + "id": 6996, + "name": "dom", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Element to render." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/types.ts", + "qualifiedName": "BasicAcceptedElems" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + ], + "name": "BasicAcceptedElems", + "package": "cheerio" + } + }, + { + "id": 6997, + "name": "options", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Options for the renderer." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/options.ts", + "qualifiedName": "CheerioOptions" + }, + "name": "CheerioOptions", + "package": "cheerio" + } + } + ], + "type": { + "type": "intrinsic", + "name": "string" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.html" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.html" + } + }, + { + "id": 7027, + "name": "merge", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 110, + "character": 24 + } + ], + "signatures": [ + { + "id": 7028, + "name": "merge", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "$.merge()." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "code", + "text": "`arr1`" + }, + { + "kind": "text", + "text": ", with elements of " + }, + { + "kind": "code", + "text": "`arr2`" + }, + { + "kind": "text", + "text": " inserted." + } + ] + }, + { + "tag": "@alias", + "content": [ + { + "kind": "text", + "text": "Cheerio.merge" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/jQuery.merge/", + "target": "https://api.jquery.com/jQuery.merge/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 110, + "character": 24 + } + ], + "typeParameters": [ + { + "id": 7029, + "name": "T", + "variant": "typeParam", + "kind": 131072, + "flags": { + "isExternal": true + } + } + ], + "parameters": [ + { + "id": 7030, + "name": "arr1", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "First array." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/cheerio/src/static.ts", + "qualifiedName": "Writable" + }, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7029, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + ], + "name": "Writable", + "package": "cheerio" + } + }, + { + "id": 7031, + "name": "arr2", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Second array." + } + ] + }, + "type": { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7029, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + } + } + ], + "type": { + "type": "union", + "types": [ + { + "type": "reference", + "target": { + "sourceFileName": "node_modules/typescript/lib/lib.es5.d.ts", + "qualifiedName": "ArrayLike" + }, + "typeArguments": [ + { + "type": "reference", + "target": 7029, + "name": "T", + "package": "cheerio", + "refersToTypeParameter": true + } + ], + "name": "ArrayLike", + "package": "typescript" + }, + { + "type": "intrinsic", + "name": "undefined" + } + ] + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.merge" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.merge" + } + }, + { + "id": 7006, + "name": "parseHTML", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 57, + "character": 24 + }, + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 58, + "character": 24 + } + ], + "signatures": [ + { + "id": 7007, + "name": "parseHTML", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Parses a string into an array of DOM nodes. The " + }, + { + "kind": "code", + "text": "`context`" + }, + { + "kind": "text", + "text": " argument has no\nmeaning for Cheerio, but it is maintained for API compatibility with jQuery." + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "The parsed DOM." + } + ] + }, + { + "tag": "@alias", + "content": [ + { + "kind": "text", + "text": "Cheerio.parseHTML" + } + ] + }, + { + "tag": "@see", + "content": [ + { + "kind": "inline-tag", + "tag": "@link", + "text": "https://api.jquery.com/jQuery.parseHTML/", + "target": "https://api.jquery.com/jQuery.parseHTML/" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 57, + "character": 24 + } + ], + "parameters": [ + { + "id": 7008, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + }, + { + "id": 7009, + "name": "data", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Markup that will be parsed." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "string" + } + }, + { + "id": 7010, + "name": "context", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Will be ignored. If it is a boolean it will be used as the\n value of " + }, + { + "kind": "code", + "text": "`keepScripts`" + }, + { + "kind": "text", + "text": "." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "unknown" + } + }, + { + "id": 7011, + "name": "keepScripts", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "If false all scripts will be removed." + } + ] + }, + "type": { + "type": "intrinsic", + "name": "boolean" + } + } + ], + "type": { + "type": "array", + "elementType": { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "AnyNode" + }, + "name": "AnyNode", + "package": "domhandler" + } + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.parseHTML" + } + }, + { + "id": 7012, + "name": "parseHTML", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 58, + "character": 24 + } + ], + "parameters": [ + { + "id": 7013, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + }, + { + "id": 7014, + "name": "data", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true, + "isOptional": true + }, + "type": { + "type": "union", + "types": [ + { + "type": "literal", + "value": null + }, + { + "type": "literal", + "value": "" + } + ] + } + } + ], + "type": { + "type": "literal", + "value": null + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.parseHTML" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.parseHTML" + } + }, + { + "id": 7015, + "name": "root", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 74, + "character": 24 + } + ], + "signatures": [ + { + "id": 7016, + "name": "root", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Sometimes you need to work with the top-level root element. To query it, you\ncan use " + }, + { + "kind": "code", + "text": "`$.root()`" + }, + { + "kind": "text", + "text": "." + } + ], + "blockTags": [ + { + "tag": "@example", + "content": [ + { + "kind": "code", + "text": "```js\n$.root().append('
        ').html();\n//=>
          ...
          \n```" + } + ] + }, + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "Cheerio instance wrapping the root node." + } + ] + }, + { + "tag": "@alias", + "content": [ + { + "kind": "text", + "text": "Cheerio.root" + } + ] + } + ] + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 74, + "character": 24 + } + ], + "parameters": [ + { + "id": 7017, + "name": "this", + "variant": "param", + "kind": 32768, + "flags": { + "isExternal": true + }, + "type": { + "type": "reference", + "target": 6980, + "name": "CheerioAPI", + "package": "cheerio" + } + } + ], + "type": { + "type": "reference", + "target": 7039, + "typeArguments": [ + { + "type": "reference", + "target": { + "sourceFileName": "../node_modules/domhandler/src/node.ts", + "qualifiedName": "Document" + }, + "name": "Document", + "package": "domhandler" + } + ], + "name": "Cheerio", + "package": "cheerio" + }, + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.root" + } + } + ], + "inheritedFrom": { + "type": "reference", + "target": -1, + "name": "StaticType.root" + } + }, + { + "id": 7002, + "name": "text", + "variant": "declaration", + "kind": 2048, + "flags": { + "isExternal": true, + "isInherited": true + }, + "sources": [ + { + "fileName": "node_modules/cheerio/dist/esm/static.d.ts", + "line": 43, + "character": 24 + } + ], + "signatures": [ + { + "id": 7003, + "name": "text", + "variant": "signature", + "kind": 4096, + "flags": { + "isExternal": true, + "isInherited": true + }, + "comment": { + "summary": [ + { + "kind": "text", + "text": "Render the document as text.\n\nThis returns the " + }, + { + "kind": "code", + "text": "`textContent`" + }, + { + "kind": "text", + "text": " of the passed elements. The result will\ninclude the contents of " + }, + { + "kind": "code", + "text": "`