Skip to content

refactor: adapt SDK Configuration for crawlee Configuration redesign #583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
B4nan wants to merge 2 commits into
base: v4
Choose a base branch
from
Open

refactor: adapt SDK Configuration for crawlee Configuration redesign #583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
392dcea
feat: redesign Configuration class for v4
B4nan Mar 13, 2026
18a23d9
fix: update configuration integration for crawlee Configuration redesign
B4nan Mar 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion packages/apify/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
"ow": "^2.0.0",
"semver": "^7.7.2",
"tslib": "^2.8.1",
"ws": "^8.18.2"
"ws": "^8.18.2",
"zod": "^3.24.0 || ^4.0.0"
}
}
73 changes: 30 additions & 43 deletions packages/apify/src/actor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
import { createPrivateKey } from 'node:crypto';

import type {
ConfigurationOptions,
EventManager,
EventTypeName,
IStorage,
RecordOptions,
UseStateOptions,
} from '@crawlee/core';
import {
Configuration as CoreConfiguration,
Dataset,
EventType,
purgeDefaultStorages,
RequestQueue,
serviceLocator,
StorageManager,
} from '@crawlee/core';
import type {
Expand Down Expand Up @@ -46,6 +45,7 @@ import { addTimeoutToPromise } from '@apify/timeout';

import type { ChargeOptions, ChargeResult } from './charging.js';
import { ChargingManager } from './charging.js';
import type { ConfigurationOptions } from './configuration.js';
import { Configuration } from './configuration.js';
import { KeyValueStore } from './key_value_store.js';
import { PlatformEventManager } from './platform_event_manager.js';
Expand Down Expand Up @@ -490,19 +490,19 @@ export class Actor<Data extends Dictionary = Dictionary> {
printOutdatedSdkWarning();

// reset global config instance to respect APIFY_ prefixed env vars
CoreConfiguration.globalConfig = Configuration.getGlobalConfig();
serviceLocator.setConfiguration(Configuration.getGlobalConfig());

if (this.isAtHome()) {
this.config.set('availableMemoryRatio', 1);
this.config.set('disableBrowserSandbox', true); // for browser launcher, adds `--no-sandbox` to args
this.config.useStorageClient(this.apifyClient);
this.config.useEventManager(this.eventManager);
// availableMemoryRatio and disableBrowserSandbox are now set via
// conditional defaults in the Configuration constructor (isAtHome check)
serviceLocator.setStorageClient(this.apifyClient);
serviceLocator.setEventManager(this.eventManager);
} else if (options.storage) {
this.config.useStorageClient(options.storage);
serviceLocator.setStorageClient(options.storage);
}

// Init the event manager the config uses
await this.config.getEventManager().init();
await serviceLocator.getEventManager().init();
log.debug(`Events initialized`);

await purgeDefaultStorages({
Expand Down Expand Up @@ -534,8 +534,8 @@ export class Actor<Data extends Dictionary = Dictionary> {
options.exit ??= true;
options.exitCode ??= EXIT_CODES.SUCCESS;
options.timeoutSecs ??= 30;
const client = this.config.getStorageClient();
const events = this.config.getEventManager();
const client = serviceLocator.getStorageClient();
const events = serviceLocator.getEventManager();

// Close the event manager and emit the final PERSIST_STATE event
await events.close();
Expand Down Expand Up @@ -601,14 +601,14 @@ export class Actor<Data extends Dictionary = Dictionary> {
* @ignore
*/
on(event: EventTypeName, listener: (...args: any[]) => any): void {
this.config.getEventManager().on(event, listener);
serviceLocator.getEventManager().on(event, listener);
}

/**
* @ignore
*/
off(event: EventTypeName, listener?: (...args: any[]) => any): void {
this.config.getEventManager().off(event, listener);
serviceLocator.getEventManager().off(event, listener);
}

/**
Expand Down Expand Up @@ -776,12 +776,10 @@ export class Actor<Data extends Dictionary = Dictionary> {
}

const {
customAfterSleepMillis = this.config.get(
'metamorphAfterSleepMillis',
),
customAfterSleepMillis = this.config.metamorphAfterSleepMillis,
...metamorphOpts
} = options;
const runId = this.config.get('actorRunId')!;
const runId = this.config.actorRunId!;
await this.apifyClient
.run(runId)
.metamorph(targetActorId, input, metamorphOpts);
Expand Down Expand Up @@ -815,27 +813,24 @@ export class Actor<Data extends Dictionary = Dictionary> {
this.isRebooting = true;

// Waiting for all the listeners to finish, as `.reboot()` kills the container.
const eventManager = serviceLocator.getEventManager();
await Promise.all([
// `persistState` for individual RequestLists, RequestQueue... instances to be persisted
...this.config
.getEventManager()
...eventManager
.listeners(EventType.PERSIST_STATE)
.map(async (x) => x()),
.map(async (x: (...args: any[]) => any) => x()),
// `migrating` to pause Apify crawlers
...this.config
.getEventManager()
...eventManager
.listeners(EventType.MIGRATING)
.map(async (x) => x()),
.map(async (x: (...args: any[]) => any) => x()),
]);

const runId = this.config.get('actorRunId')!;
const runId = this.config.actorRunId!;
await this.apifyClient.run(runId).reboot();

// Wait some time for container to be stopped.
const {
customAfterSleepMillis = this.config.get(
'metamorphAfterSleepMillis',
),
customAfterSleepMillis = this.config.metamorphAfterSleepMillis,
} = options;
await sleep(customAfterSleepMillis);
}
Expand Down Expand Up @@ -873,7 +868,7 @@ export class Actor<Data extends Dictionary = Dictionary> {
return undefined;
}

const runId = this.config.get('actorRunId')!;
const runId = this.config.actorRunId!;
if (!runId) {
throw new Error(
`Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`,
Expand Down Expand Up @@ -924,7 +919,7 @@ export class Actor<Data extends Dictionary = Dictionary> {
break;
}

const client = this.config.getStorageClient();
const client = serviceLocator.getStorageClient();

// just to be sure, this should be fast
await addTimeoutToPromise(
Expand All @@ -937,7 +932,7 @@ export class Actor<Data extends Dictionary = Dictionary> {
'Setting status message timed out after 1s',
).catch((e) => log.warning(e.message));

const runId = this.config.get('actorRunId')!;
const runId = this.config.actorRunId!;

if (runId) {
// just to be sure, this should be fast
Expand Down Expand Up @@ -1213,13 +1208,9 @@ export class Actor<Data extends Dictionary = Dictionary> {
async getInput<T = Dictionary | string | Buffer>(): Promise<T | null> {
this._ensureActorInit('getInput');

const inputSecretsPrivateKeyFile = this.config.get(
'inputSecretsPrivateKeyFile',
);
const inputSecretsPrivateKeyPassphrase = this.config.get(
'inputSecretsPrivateKeyPassphrase',
);
const input = await this.getValue<T>(this.config.get('inputKey'));
const { inputSecretsPrivateKeyFile } = this.config;
const { inputSecretsPrivateKeyPassphrase } = this.config;
const input = await this.getValue<T>(this.config.inputKey);
if (
ow.isValid(input, ow.object.nonEmpty) &&
inputSecretsPrivateKeyFile &&
Expand Down Expand Up @@ -1476,18 +1467,14 @@ export class Actor<Data extends Dictionary = Dictionary> {
* @ignore
*/
newClient(options: ApifyClientOptions = {}): ApifyClient {
const { storageDir, ...storageClientOptions } = this.config.get(
'storageClientOptions',
) as Dictionary;
const { apifyVersion, crawleeVersion } = getSystemInfo();
return new ApifyClient({
baseUrl: this.config.get('apiBaseUrl'),
token: this.config.get('token'),
baseUrl: this.config.apiBaseUrl,
token: this.config.token,
userAgentSuffix: [
`SDK/${apifyVersion}`,
`Crawlee/${crawleeVersion}`,
],
...storageClientOptions,
...options, // allow overriding the instance configuration
});
}
Expand Down
13 changes: 6 additions & 7 deletions packages/apify/src/charging.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,19 @@ export class ChargingManager {
private apifyClient: ApifyClient;

constructor(configuration: Configuration, apifyClient: ApifyClient) {
this.maxTotalChargeUsd =
configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string
this.isAtHome = configuration.get('isAtHome');
this.actorRunId = configuration.get('actorRunId');
this.purgeChargingLogDataset = configuration.get('purgeOnStart');
this.useChargingLogDataset = configuration.get('useChargingLogDataset');
this.maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string
this.isAtHome = !!configuration.isAtHome;
this.actorRunId = configuration.actorRunId;
this.purgeChargingLogDataset = configuration.purgeOnStart;
this.useChargingLogDataset = configuration.useChargingLogDataset;

if (this.useChargingLogDataset && this.isAtHome) {
throw new Error(
'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment',
);
}

if (configuration.get('testPayPerEvent')) {
if (configuration.testPayPerEvent) {
if (this.isAtHome) {
throw new Error(
'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment',
Expand Down
Loading
Loading