diff --git a/package-lock.json b/package-lock.json index 1b05834ad6..1f4a8a966c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20046,7 +20046,6 @@ "version": "3.25.28", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.28.tgz", "integrity": "sha512-/nt/67WYKnr5by3YS7LroZJbtcCBurDKKPBPWWzaxvVCGuG/NOsiKkrjoOhI8mJ+SQUXEbUzeB3S+6XDUEEj7Q==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -20195,7 +20194,8 @@ "ow": "^2.0.0", "semver": "^7.7.2", "tslib": "^2.8.1", - "ws": "^8.18.2" + "ws": "^8.18.2", + "zod": "^3.24.0 || ^4.0.0" }, "engines": { "node": ">=22.0.0" diff --git a/packages/apify/package.json b/packages/apify/package.json index f3ddcf0be1..ebb8146b6d 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -62,6 +62,7 @@ "ow": "^2.0.0", "semver": "^7.7.2", "tslib": "^2.8.1", - "ws": "^8.18.2" + "ws": "^8.18.2", + "zod": "^3.24.0 || ^4.0.0" } } diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 627cacaefb..86fd1b9dce 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -1,7 +1,6 @@ import { createPrivateKey } from 'node:crypto'; import type { - ConfigurationOptions, EventManager, EventTypeName, IStorage, @@ -9,11 +8,11 @@ import type { UseStateOptions, } from '@crawlee/core'; import { - Configuration as CoreConfiguration, Dataset, EventType, purgeDefaultStorages, RequestQueue, + serviceLocator, StorageManager, } from '@crawlee/core'; import type { @@ -46,6 +45,7 @@ import { addTimeoutToPromise } from '@apify/timeout'; import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager } from './charging.js'; +import type { ConfigurationOptions } from './configuration.js'; import { Configuration } from './configuration.js'; import { KeyValueStore } from './key_value_store.js'; import { PlatformEventManager } from './platform_event_manager.js'; @@ -490,19 +490,19 @@ export class Actor { printOutdatedSdkWarning(); // reset global config instance to respect APIFY_ prefixed env vars - CoreConfiguration.globalConfig = Configuration.getGlobalConfig(); + serviceLocator.setConfiguration(Configuration.getGlobalConfig()); if (this.isAtHome()) { - this.config.set('availableMemoryRatio', 1); - this.config.set('disableBrowserSandbox', true); // for browser launcher, adds `--no-sandbox` to args - this.config.useStorageClient(this.apifyClient); - this.config.useEventManager(this.eventManager); + // availableMemoryRatio and disableBrowserSandbox are now set via + // conditional defaults in the Configuration constructor (isAtHome check) + serviceLocator.setStorageClient(this.apifyClient); + serviceLocator.setEventManager(this.eventManager); } else if (options.storage) { - this.config.useStorageClient(options.storage); + serviceLocator.setStorageClient(options.storage); } // Init the event manager the config uses - await this.config.getEventManager().init(); + await serviceLocator.getEventManager().init(); log.debug(`Events initialized`); await purgeDefaultStorages({ @@ -534,8 +534,8 @@ export class Actor { options.exit ??= true; options.exitCode ??= EXIT_CODES.SUCCESS; options.timeoutSecs ??= 30; - const client = this.config.getStorageClient(); - const events = this.config.getEventManager(); + const client = serviceLocator.getStorageClient(); + const events = serviceLocator.getEventManager(); // Close the event manager and emit the final PERSIST_STATE event await events.close(); @@ -601,14 +601,14 @@ export class Actor { * @ignore */ on(event: EventTypeName, listener: (...args: any[]) => any): void { - this.config.getEventManager().on(event, listener); + serviceLocator.getEventManager().on(event, listener); } /** * @ignore */ off(event: EventTypeName, listener?: (...args: any[]) => any): void { - this.config.getEventManager().off(event, listener); + serviceLocator.getEventManager().off(event, listener); } /** @@ -776,12 +776,10 @@ export class Actor { } const { - customAfterSleepMillis = this.config.get( - 'metamorphAfterSleepMillis', - ), + customAfterSleepMillis = this.config.metamorphAfterSleepMillis, ...metamorphOpts } = options; - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; await this.apifyClient .run(runId) .metamorph(targetActorId, input, metamorphOpts); @@ -815,27 +813,24 @@ export class Actor { this.isRebooting = true; // Waiting for all the listeners to finish, as `.reboot()` kills the container. + const eventManager = serviceLocator.getEventManager(); await Promise.all([ // `persistState` for individual RequestLists, RequestQueue... instances to be persisted - ...this.config - .getEventManager() + ...eventManager .listeners(EventType.PERSIST_STATE) - .map(async (x) => x()), + .map(async (x: (...args: any[]) => any) => x()), // `migrating` to pause Apify crawlers - ...this.config - .getEventManager() + ...eventManager .listeners(EventType.MIGRATING) - .map(async (x) => x()), + .map(async (x: (...args: any[]) => any) => x()), ]); - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; await this.apifyClient.run(runId).reboot(); // Wait some time for container to be stopped. const { - customAfterSleepMillis = this.config.get( - 'metamorphAfterSleepMillis', - ), + customAfterSleepMillis = this.config.metamorphAfterSleepMillis, } = options; await sleep(customAfterSleepMillis); } @@ -873,7 +868,7 @@ export class Actor { return undefined; } - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; if (!runId) { throw new Error( `Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`, @@ -924,7 +919,7 @@ export class Actor { break; } - const client = this.config.getStorageClient(); + const client = serviceLocator.getStorageClient(); // just to be sure, this should be fast await addTimeoutToPromise( @@ -937,7 +932,7 @@ export class Actor { 'Setting status message timed out after 1s', ).catch((e) => log.warning(e.message)); - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; if (runId) { // just to be sure, this should be fast @@ -1213,13 +1208,9 @@ export class Actor { async getInput(): Promise { this._ensureActorInit('getInput'); - const inputSecretsPrivateKeyFile = this.config.get( - 'inputSecretsPrivateKeyFile', - ); - const inputSecretsPrivateKeyPassphrase = this.config.get( - 'inputSecretsPrivateKeyPassphrase', - ); - const input = await this.getValue(this.config.get('inputKey')); + const { inputSecretsPrivateKeyFile } = this.config; + const { inputSecretsPrivateKeyPassphrase } = this.config; + const input = await this.getValue(this.config.inputKey); if ( ow.isValid(input, ow.object.nonEmpty) && inputSecretsPrivateKeyFile && @@ -1476,18 +1467,14 @@ export class Actor { * @ignore */ newClient(options: ApifyClientOptions = {}): ApifyClient { - const { storageDir, ...storageClientOptions } = this.config.get( - 'storageClientOptions', - ) as Dictionary; const { apifyVersion, crawleeVersion } = getSystemInfo(); return new ApifyClient({ - baseUrl: this.config.get('apiBaseUrl'), - token: this.config.get('token'), + baseUrl: this.config.apiBaseUrl, + token: this.config.token, userAgentSuffix: [ `SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`, ], - ...storageClientOptions, ...options, // allow overriding the instance configuration }); } diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 5ac7777846..e6100f5652 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -87,12 +87,11 @@ export class ChargingManager { private apifyClient: ApifyClient; constructor(configuration: Configuration, apifyClient: ApifyClient) { - this.maxTotalChargeUsd = - configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string - this.isAtHome = configuration.get('isAtHome'); - this.actorRunId = configuration.get('actorRunId'); - this.purgeChargingLogDataset = configuration.get('purgeOnStart'); - this.useChargingLogDataset = configuration.get('useChargingLogDataset'); + this.maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string + this.isAtHome = !!configuration.isAtHome; + this.actorRunId = configuration.actorRunId; + this.purgeChargingLogDataset = configuration.purgeOnStart; + this.useChargingLogDataset = configuration.useChargingLogDataset; if (this.useChargingLogDataset && this.isAtHome) { throw new Error( @@ -100,7 +99,7 @@ export class ChargingManager { ); } - if (configuration.get('testPayPerEvent')) { + if (configuration.testPayPerEvent) { if (this.isAtHome) { throw new Error( 'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment', diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index b8dfacd42c..4228f5d93b 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -1,7 +1,15 @@ -import type { ConfigurationOptions as CoreConfigurationOptions } from '@crawlee/core'; -import { Configuration as CoreConfiguration } from '@crawlee/core'; +/* eslint-disable no-use-before-define */ +import { AsyncLocalStorage } from 'node:async_hooks'; + +import type { ConfigField, FieldsInput, FieldsOutput } from '@crawlee/core'; +import { + coerceBoolean, + Configuration as CoreConfiguration, + crawleeConfigFields, + field, +} from '@crawlee/core'; +import { z } from 'zod'; -import type { META_ORIGINS } from '@apify/consts'; import { ACTOR_ENV_VARS, APIFY_ENV_VARS, @@ -9,37 +17,194 @@ import { LOCAL_APIFY_ENV_VARS, } from '@apify/consts'; -export interface ConfigurationOptions extends CoreConfigurationOptions { - metamorphAfterSleepMillis?: number; - actorEventsWsUrl?: string; - token?: string; - actorId?: string; - actorRunId?: string; - actorTaskId?: string; - apiBaseUrl?: string; - // apiBaseUrl is the internal API URL, accessible only within the platform(private network), - // while apiPublicBaseUrl is the public API URL, available externally(through internet). - apiPublicBaseUrl?: string; - containerPort?: number; - containerUrl?: string; - proxyHostname?: string; - proxyPassword?: string; - proxyPort?: number; - proxyStatusUrl?: string; - /** - * @deprecated use `containerPort` instead - */ - standbyPort?: number; - standbyUrl?: string; - isAtHome?: boolean; - userId?: string; - inputSecretsPrivateKeyPassphrase?: string; - inputSecretsPrivateKeyFile?: string; - maxTotalChargeUsd?: number; - metaOrigin?: (typeof META_ORIGINS)[keyof typeof META_ORIGINS]; - testPayPerEvent?: boolean; - useChargingLogDataset?: boolean; -} +const coerceNumber = z.preprocess((val) => { + if (typeof val === 'string') return Number(val); + return val; +}, z.number()); + +// --- isAtHome check (simple env var presence) --- +const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME]; + +// --- Apify config field definitions --- + +export const apifyConfigFields = { + // Inherit all crawlee fields, overriding env vars where the SDK supports ACTOR_/APIFY_ aliases + ...crawleeConfigFields, + + // Override crawlee fields with ACTOR_/APIFY_ env var aliases + defaultDatasetId: field( + z + .string() + .default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID]), + [ + ACTOR_ENV_VARS.DEFAULT_DATASET_ID, + 'APIFY_DEFAULT_DATASET_ID', + 'CRAWLEE_DEFAULT_DATASET_ID', + ], + ), + defaultKeyValueStoreId: field( + z + .string() + .default( + LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID], + ), + [ + ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, + 'APIFY_DEFAULT_KEY_VALUE_STORE_ID', + 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID', + ], + ), + defaultRequestQueueId: field( + z + .string() + .default( + LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], + ), + [ + ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, + 'APIFY_DEFAULT_REQUEST_QUEUE_ID', + 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID', + ], + ), + inputKey: field(z.string().default('INPUT'), [ + ACTOR_ENV_VARS.INPUT_KEY, + 'APIFY_INPUT_KEY', + 'CRAWLEE_INPUT_KEY', + ]), + memoryMbytes: field(coerceNumber.optional(), [ + ACTOR_ENV_VARS.MEMORY_MBYTES, + 'APIFY_MEMORY_MBYTES', + 'CRAWLEE_MEMORY_MBYTES', + ]), + availableMemoryRatio: field(coerceNumber.default(isAtHome ? 1 : 0.25), [ + 'CRAWLEE_AVAILABLE_MEMORY_RATIO', + 'APIFY_AVAILABLE_MEMORY_RATIO', + ]), + disableBrowserSandbox: field( + isAtHome ? coerceBoolean.default(true) : coerceBoolean.optional(), + ['CRAWLEE_DISABLE_BROWSER_SANDBOX', 'APIFY_DISABLE_BROWSER_SANDBOX'], + ), + persistStateIntervalMillis: field(coerceNumber.default(60_000), [ + 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', + 'APIFY_PERSIST_STATE_INTERVAL_MILLIS', + 'APIFY_TEST_PERSIST_INTERVAL_MILLIS', + ]), + headless: field(coerceBoolean.default(true), [ + 'CRAWLEE_HEADLESS', + 'APIFY_HEADLESS', + ]), + xvfb: field(coerceBoolean.default(false), ['CRAWLEE_XVFB', 'APIFY_XVFB']), + chromeExecutablePath: field(z.string().optional(), [ + 'CRAWLEE_CHROME_EXECUTABLE_PATH', + 'APIFY_CHROME_EXECUTABLE_PATH', + ]), + defaultBrowserPath: field(z.string().optional(), [ + 'CRAWLEE_DEFAULT_BROWSER_PATH', + 'APIFY_DEFAULT_BROWSER_PATH', + ]), + purgeOnStart: field(coerceBoolean.default(true), [ + 'CRAWLEE_PURGE_ON_START', + 'APIFY_PURGE_ON_START', + ]), + + // Apify-specific fields + metamorphAfterSleepMillis: field( + coerceNumber.default(300_000), + 'APIFY_METAMORPH_AFTER_SLEEP_MILLIS', + ), + actorEventsWsUrl: field(z.string().optional(), [ + ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL, + 'APIFY_ACTOR_EVENTS_WS_URL', + ]), + token: field(z.string().optional(), 'APIFY_TOKEN'), + actorId: field(z.string().optional(), [ + ACTOR_ENV_VARS.ID, + 'APIFY_ACTOR_ID', + ]), + actorRunId: field(z.string().optional(), [ + ACTOR_ENV_VARS.RUN_ID, + 'APIFY_ACTOR_RUN_ID', + ]), + actorTaskId: field(z.string().optional(), [ + ACTOR_ENV_VARS.TASK_ID, + 'APIFY_ACTOR_TASK_ID', + ]), + apiBaseUrl: field( + z.string().default('https://api.apify.com'), + 'APIFY_API_BASE_URL', + ), + apiPublicBaseUrl: field( + z.string().default('https://api.apify.com'), + 'APIFY_API_PUBLIC_BASE_URL', + ), + containerPort: field( + coerceNumber.default( + +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_PORT], + ), + [ACTOR_ENV_VARS.WEB_SERVER_PORT, 'APIFY_CONTAINER_PORT'], + ), + containerUrl: field( + z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_URL]), + [ACTOR_ENV_VARS.WEB_SERVER_URL, 'APIFY_CONTAINER_URL'], + ), + proxyHostname: field( + z.string().default(LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME]), + 'APIFY_PROXY_HOSTNAME', + ), + proxyPassword: field(z.string().optional(), 'APIFY_PROXY_PASSWORD'), + proxyPort: field( + coerceNumber.default(+LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT]), + 'APIFY_PROXY_PORT', + ), + proxyStatusUrl: field( + z.string().default('http://proxy.apify.com'), + 'APIFY_PROXY_STATUS_URL', + ), + /** @deprecated use `containerPort` instead */ + standbyPort: field( + coerceNumber.default( + +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.STANDBY_PORT], + ), + ACTOR_ENV_VARS.STANDBY_PORT, + ), + standbyUrl: field(z.string().optional(), ACTOR_ENV_VARS.STANDBY_URL), + isAtHome: field(coerceBoolean.optional(), 'APIFY_IS_AT_HOME'), + userId: field(z.string().optional(), 'APIFY_USER_ID'), + inputSecretsPrivateKeyPassphrase: field( + z.string().optional(), + 'APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE', + ), + inputSecretsPrivateKeyFile: field( + z.string().optional(), + 'APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE', + ), + maxTotalChargeUsd: field( + coerceNumber.optional(), + ACTOR_ENV_VARS.MAX_TOTAL_CHARGE_USD, + ), + metaOrigin: field(z.string().optional(), 'APIFY_META_ORIGIN'), + testPayPerEvent: field( + coerceBoolean.default(false), + 'ACTOR_TEST_PAY_PER_EVENT', + ), + useChargingLogDataset: field( + coerceBoolean.default(false), + 'ACTOR_USE_CHARGING_LOG_DATASET', + ), +}; + +// --- Type utilities --- + +export type ApifyConfigurationInput = FieldsInput; +export type ApifyResolvedConfigValues = FieldsOutput; + +/** @deprecated Use {@link ApifyConfigurationInput} instead. */ +export type ConfigurationOptions = ApifyConfigurationInput; + +// --- Configuration class --- + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type, @typescript-eslint/no-unsafe-declaration-merging +export interface Configuration extends ApifyResolvedConfigValues {} /** * `Configuration` is a value object holding the SDK configuration. We can use it in two ways: @@ -48,38 +213,34 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { * * ```javascript * import { Actor } from 'apify'; - * import { BasicCrawler } from 'crawlee'; * * const sdk = new Actor({ token: '123' }); - * console.log(sdk.config.get('token')); // '123' - * - * const crawler = new BasicCrawler({ - * // ... crawler options - * }, sdk.config); + * console.log(sdk.config.token); // '123' * ``` * * 2. To get the global configuration (singleton instance). It will respect the environment variables. * * ```javascript - * import { BasicCrawler, Configuration } from 'crawlee'; + * import { Configuration } from 'apify'; * - * // Get the global configuration * const config = Configuration.getGlobalConfig(); - * // Set the 'persistStateIntervalMillis' option - * // of global configuration to 30 seconds - * config.set('persistStateIntervalMillis', 30_000); - * - * // No need to pass the configuration to the crawler, - * // as it's using the global configuration by default - * const crawler = new BasicCrawler(); + * console.log(config.headless); + * console.log(config.persistStateIntervalMillis); * ``` * + * Configuration is immutable — values are set via the constructor and cannot be changed afterwards. + * The priority order for resolving values is (highest to lowest): + * + * ```text + * constructor options > environment variables > crawlee.json > schema defaults + * ``` + * * ## Supported Configuration Options * * Key | Environment Variable | Default Value * ---|---|--- * `memoryMbytes` | `ACTOR_MEMORY_MBYTES` | - - * `headless` | `APIFY_HEADLESS` | - + * `headless` | `APIFY_HEADLESS` | `true` * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3` * `token` | `APIFY_TOKEN` | - * `isAtHome` | `APIFY_IS_AT_HOME` | - @@ -112,126 +273,19 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { * `chromeExecutablePath` | `APIFY_CHROME_EXECUTABLE_PATH` | - * `defaultBrowserPath` | `APIFY_DEFAULT_BROWSER_PATH` | - */ +// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging export class Configuration extends CoreConfiguration { - /** @inheritDoc */ - // eslint-disable-next-line no-use-before-define -- Self-reference - static override globalConfig?: Configuration; - - // maps environment variables to config keys (e.g. `APIFY_MEMORY_MBYTES` to `memoryMbytes`) - protected static override ENV_MAP = { - // regular crawlee env vars are also supported - ...CoreConfiguration.ENV_MAP, - - // support crawlee env vars prefixed with `APIFY_` too - APIFY_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio', - APIFY_PURGE_ON_START: 'purgeOnStart', - APIFY_MEMORY_MBYTES: 'memoryMbytes', - APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId', - APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId', - APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId', - APIFY_INPUT_KEY: 'inputKey', - APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis', - APIFY_HEADLESS: 'headless', - APIFY_XVFB: 'xvfb', - APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath', - APIFY_DEFAULT_BROWSER_PATH: 'defaultBrowserPath', - APIFY_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox', - - // as well as apify specific ones - APIFY_TOKEN: 'token', - APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis', - APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis', // for BC, seems to be unused - APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl', - APIFY_ACTOR_ID: 'actorId', - APIFY_API_BASE_URL: 'apiBaseUrl', - APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl', - APIFY_IS_AT_HOME: 'isAtHome', - APIFY_ACTOR_RUN_ID: 'actorRunId', - APIFY_ACTOR_TASK_ID: 'actorTaskId', - APIFY_CONTAINER_PORT: 'containerPort', - APIFY_CONTAINER_URL: 'containerUrl', - APIFY_USER_ID: 'userId', - APIFY_PROXY_HOSTNAME: 'proxyHostname', - APIFY_PROXY_PASSWORD: 'proxyPassword', - APIFY_PROXY_STATUS_URL: 'proxyStatusUrl', - APIFY_PROXY_PORT: 'proxyPort', - APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE: 'inputSecretsPrivateKeyFile', - APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE: - 'inputSecretsPrivateKeyPassphrase', - APIFY_META_ORIGIN: 'metaOrigin', - - // Actor env vars - ACTOR_DEFAULT_DATASET_ID: 'defaultDatasetId', - ACTOR_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId', - ACTOR_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId', - ACTOR_EVENTS_WEBSOCKET_URL: 'actorEventsWsUrl', - ACTOR_ID: 'actorId', - ACTOR_INPUT_KEY: 'inputKey', - ACTOR_MEMORY_MBYTES: 'memoryMbytes', - ACTOR_RUN_ID: 'actorRunId', - ACTOR_STANDBY_PORT: 'standbyPort', - ACTOR_STANDBY_URL: 'standbyUrl', - ACTOR_TASK_ID: 'actorTaskId', - ACTOR_WEB_SERVER_PORT: 'containerPort', - ACTOR_WEB_SERVER_URL: 'containerUrl', - ACTOR_MAX_TOTAL_CHARGE_USD: 'maxTotalChargeUsd', - ACTOR_TEST_PAY_PER_EVENT: 'testPayPerEvent', - ACTOR_USE_CHARGING_LOG_DATASET: 'useChargingLogDataset', - }; - - protected static override INTEGER_VARS = [ - ...CoreConfiguration.INTEGER_VARS, - 'proxyPort', - 'containerPort', - 'metamorphAfterSleepMillis', - 'maxTotalChargeUsd', - ]; - - protected static override BOOLEAN_VARS = [ - ...CoreConfiguration.BOOLEAN_VARS, - 'isAtHome', - 'testPayPerEvent', - 'useChargingLogDataset', - ]; + /** @internal */ + static storage = new AsyncLocalStorage(); - protected static override DEFAULTS = { - ...CoreConfiguration.DEFAULTS, - defaultKeyValueStoreId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID], - defaultDatasetId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID], - defaultRequestQueueId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], - inputKey: 'INPUT', - apiBaseUrl: 'https://api.apify.com', - apiPublicBaseUrl: 'https://api.apify.com', - proxyStatusUrl: 'http://proxy.apify.com', - proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME], - proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT], - containerPort: +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_PORT], - containerUrl: LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_URL], - standbyPort: +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.STANDBY_PORT], - metamorphAfterSleepMillis: 300e3, - persistStateIntervalMillis: 60e3, // This value is mentioned in jsdoc in `events.js`, if you update it here, update it there too. - testPayPerEvent: false, - useChargingLogDataset: false, - }; + /** @internal */ + static globalConfig?: Configuration; - /** - * @inheritDoc - */ - override get< - T extends keyof ConfigurationOptions, - U extends ConfigurationOptions[T], - >(key: T, defaultValue?: U): U { - return super.get(key as keyof CoreConfigurationOptions, defaultValue); - } + protected static override fields: Record = + apifyConfigFields; - /** - * @inheritDoc - */ - override set(key: keyof ConfigurationOptions, value?: any) { - super.set(key as keyof CoreConfigurationOptions, value); + constructor(options: ApifyConfigurationInput = {}) { + super(options as any); } /** @@ -250,18 +304,7 @@ export class Configuration extends CoreConfiguration { * Resets global configuration instance. The default instance holds configuration based on env vars, * if we want to change them, we need to first reset the global state. Used mainly for testing purposes. */ - static override resetGlobalState(): void { + static resetGlobalState(): void { delete this.globalConfig; } } - -// monkey patch the core class so it respects the new options too -CoreConfiguration.getGlobalConfig = Configuration.getGlobalConfig; -// @ts-expect-error protected property -CoreConfiguration.ENV_MAP = Configuration.ENV_MAP; -// @ts-expect-error protected property -CoreConfiguration.INTEGER_VARS = Configuration.INTEGER_VARS; -// @ts-expect-error protected property -CoreConfiguration.BOOLEAN_VARS = Configuration.BOOLEAN_VARS; -// @ts-expect-error protected property -CoreConfiguration.DEFAULTS = Configuration.DEFAULTS; diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 89f2138d2c..a26a12e8f1 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -18,12 +18,12 @@ export class KeyValueStore extends CoreKeyValueStore { */ override getPublicUrl(key: string): string { const config = this.config as Configuration; - if (!config.get('isAtHome') && getPublicUrl) { + if (!config.isAtHome && getPublicUrl) { return getPublicUrl.call(this, key); } const publicUrl = new URL( - `${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`, + `${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`, ); if (this.storageObject?.urlSigningSecretKey) { diff --git a/packages/apify/src/platform_event_manager.ts b/packages/apify/src/platform_event_manager.ts index a71c5ee68e..4385eebd60 100644 --- a/packages/apify/src/platform_event_manager.ts +++ b/packages/apify/src/platform_event_manager.ts @@ -62,7 +62,7 @@ export class PlatformEventManager extends EventManager { } await super.init(); - const eventsWsUrl = this.config.get('actorEventsWsUrl'); + const eventsWsUrl = (this.config as Configuration).actorEventsWsUrl; // Locally there is no web socket to connect, so just print a log message. if (!eventsWsUrl) { diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 569ea84ae3..0d452ab645 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -205,7 +205,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { apifyProxyGroups = [], countryCode, apifyProxyCountry, - password = config.get('proxyPassword'), + password = config.proxyPassword, tieredProxyConfig, tieredProxyUrls, } = options; @@ -221,8 +221,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration { const groupsToUse = groups.length ? groups : apifyProxyGroups; const countryCodeToUse = countryCode || apifyProxyCountry; - const hostname = config.get('proxyHostname'); - const port = config.get('proxyPort'); + const hostname = config.proxyHostname; + const port = config.proxyPort; // Validation if ( @@ -438,7 +438,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { */ // TODO: Make this private protected async _setPasswordIfToken(): Promise { - const token = this.config.get('token'); + const {token} = (this.config as Configuration); if (!token) return; try { @@ -500,10 +500,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { } | undefined > { - const proxyStatusUrl = this.config.get( - 'proxyStatusUrl', - 'http://proxy.apify.com', - ); + const {proxyStatusUrl} = (this.config as Configuration); const requestOpts = { url: `${proxyStatusUrl}/?format=json`, proxyUrl: await this.newUrl(), diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 47565929ff..f1b88fc281 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -705,13 +705,14 @@ describe('Actor', () => { expect(getValueSpy).toBeCalledWith(KEY_VALUE_STORE_KEYS.INPUT); expect(val1).toBe(123); - // Uses value from config - sdk.config.set('inputKey', 'some-value'); - const val2 = await sdk.getInput(); - expect(getValueSpy).toBeCalledTimes(2); - expect(getValueSpy).toBeCalledWith('some-value'); + // Uses value from config - create a new Actor with custom inputKey + const sdk2 = new Actor({ inputKey: 'some-value' }); + const getValueSpy2 = vitest.spyOn(sdk2 as any, 'getValue'); + getValueSpy2.mockImplementation(async () => 123); + const val2 = await sdk2.getInput(); + expect(getValueSpy2).toBeCalledTimes(1); + expect(getValueSpy2).toBeCalledWith('some-value'); expect(val2).toBe(123); - sdk.config.set('inputKey', undefined); // restore defaults }); test('setValue()', async () => { @@ -1282,18 +1283,19 @@ describe('Actor', () => { }); describe('Actor.config and PPE', () => { - test('should work', async () => { - await Actor.init(); + test('empty string maxTotalChargeUsd coerces to 0, charging manager treats as Infinity', async () => { process.env.ACTOR_MAX_TOTAL_CHARGE_USD = ''; - expect(Actor.config.get('maxTotalChargeUsd')).toBe(0); + await Actor.init(); + expect(Actor.config.maxTotalChargeUsd).toBe(0); expect(Actor.getChargingManager().getMaxTotalChargeUsd()).toBe( Infinity, ); - - // the value in charging manager is cached, so we cant test that here - process.env.ACTOR_MAX_TOTAL_CHARGE_USD = '123'; - expect(Actor.config.get('maxTotalChargeUsd')).toBe(123); await Actor.exit({ exit: false }); }); + + test('numeric maxTotalChargeUsd is correctly resolved from constructor options', () => { + const sdk = new Actor({ maxTotalChargeUsd: 123 }); + expect(sdk.config.maxTotalChargeUsd).toBe(123); + }); }); }); diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index cb7804fbb7..d6ec40d3ec 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -130,7 +130,7 @@ describe('events', () => { test('should send persist state events in regular interval', async () => { const eventsReceived = []; - const interval = config.get('persistStateIntervalMillis')!; + const interval = config.persistStateIntervalMillis; events.on(EventType.PERSIST_STATE, (data) => eventsReceived.push(data)); await events.init();