fix: constructor options take precedence over env vars in Configuration (#3080)

Changes the configuration priority so that user-provided constructor options
take precedence over environment variables, while env vars still override
defaults and crawlee.json settings.

New priority: constructor options > env vars > crawlee.json > defaults

This allows users to programmatically override environment variables, e.g.,
`new Configuration({ headless: false })` now works even if CRAWLEE_HEADLESS=true.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: B4nan

docs/upgrading/upgrading_v4.md

@@ -114,3 +114,19 @@ The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public
 ## `preNavigationHooks` in `HttpCrawler` no longer accepts `gotOptions` object
 
 The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.
+
+## Configuration priority change
+
+The priority of configuration options has changed. Previously, environment variables always took precedence over constructor options:
+
+```text
+crawlee.json < constructor options < environment variables
+```
+
+Now, constructor options take the highest precedence, allowing you to programmatically override environment variables:
+
+```text
+constructor options > environment variables > crawlee.json > defaults
+```
+
+This means that if you have `CRAWLEE_HEADLESS=true` set in your environment, you can now override it by passing `new Configuration({ headless: false })`. Previously, the environment variable would always win.

packages/core/src/configuration.ts

@@ -204,12 +204,12 @@ export interface ConfigurationOptions {
  * const crawler = new BasicCrawler({ ... }, config);
  * ```
  *
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Options explicitly provided in the constructor take the highest precedence, followed by
+ * environment variables, then the `crawlee.json` file in the project root directory, and finally
+ * the default values. In other words, the precedence is:
  *
  * ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > defaults
  * ```
  *
  * ## Supported Configuration Options
@@ -296,6 +296,7 @@ export class Configuration {
     static storage = new AsyncLocalStorage<Configuration>();
 
     protected options!: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
+    protected userOptions!: Set<keyof ConfigurationOptions>;
     protected services = new Map<string, unknown>();
 
     /** @internal */
@@ -304,7 +305,7 @@ export class Configuration {
     public readonly storageManagers = new Map<Constructor, StorageManager>();
 
     /**
-     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+     * Creates new `Configuration` instance with provided options. Constructor options take precedence over env vars.
      */
     constructor(options: ConfigurationOptions = {}) {
         this.buildOptions(options);
@@ -324,12 +325,18 @@ export class Configuration {
     }
 
     /**
-     * Returns configured value. First checks the environment variables, then provided configuration,
+     * Returns configured value. First checks options explicitly provided in the constructor,
+     * then environment variables, then options from crawlee.json, and finally
      * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
      * in the above section.
      */
     get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U {
-        // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
+        // 1. Check if user explicitly provided this option (highest priority)
+        if (this.userOptions.has(key) && this.options.has(key)) {
+            return this.options.get(key) as U;
+        }
+
+        // 2. Check environment variables (second priority)
         let envValue: string | undefined;
 
         for (const [k, v] of entries(Configuration.ENV_MAP)) {
@@ -346,13 +353,13 @@ export class Configuration {
             return this._castEnvValue(key, envValue) as U;
         }
 
-        // check instance level options
+        // 3. Check options from crawlee.json (third priority)
         if (this.options.has(key)) {
             return this.options.get(key) as U;
         }
 
-        // fallback to defaults
-        return (defaultValue ?? Configuration.DEFAULTS[key as keyof typeof Configuration.DEFAULTS] ?? envValue) as U;
+        // 4. Fallback to defaults (lowest priority)
+        return (defaultValue ?? Configuration.DEFAULTS[key as keyof typeof Configuration.DEFAULTS]) as U;
     }
 
     protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean) {
@@ -492,14 +499,18 @@ export class Configuration {
     }
 
     protected buildOptions(options: ConfigurationOptions) {
-        // try to load configuration from crawlee.json as the baseline
+        // Track which options were explicitly provided by the user
+        this.userOptions = new Set(Object.keys(options) as (keyof ConfigurationOptions)[]);
+
+        // Load crawlee.json as baseline, then merge user options on top
         const path = join(process.cwd(), 'crawlee.json');
 
         if (pathExistsSync(path)) {
             try {
                 const file = readFileSync(path);
                 const optionsFromFileConfig = JSON.parse(file.toString());
-                Object.assign(options, optionsFromFileConfig);
+                // Merge file config first, then user options override
+                options = { ...optionsFromFileConfig, ...options };
             } catch {
                 // ignore
             }

packages/core/test/configuration-priority.test.ts

@@ -0,0 +1,198 @@
+import { unlinkSync,writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { Configuration } from '@crawlee/core';
+
+describe('Configuration priority', () => {
+    const originalEnv = { ...process.env };
+    const crawleeJsonPath = join(process.cwd(), 'crawlee.json');
+    let createdCrawleeJson = false;
+
+    beforeEach(() => {
+        Configuration.resetGlobalState();
+    });
+
+    afterEach(() => {
+        // Restore original environment
+        process.env = { ...originalEnv };
+        Configuration.resetGlobalState();
+
+        // Clean up crawlee.json if we created it
+        if (createdCrawleeJson) {
+            try {
+                unlinkSync(crawleeJsonPath);
+            } catch {
+                // ignore
+            }
+            createdCrawleeJson = false;
+        }
+    });
+
+    describe('constructor options take precedence over env vars', () => {
+        test('boolean option: headless', () => {
+            process.env.CRAWLEE_HEADLESS = 'true';
+            const config = new Configuration({ headless: false });
+
+            expect(config.get('headless')).toBe(false);
+        });
+
+        test('string option: defaultDatasetId', () => {
+            process.env.CRAWLEE_DEFAULT_DATASET_ID = 'env-dataset';
+            const config = new Configuration({ defaultDatasetId: 'constructor-dataset' });
+
+            expect(config.get('defaultDatasetId')).toBe('constructor-dataset');
+        });
+
+        test('integer option: memoryMbytes', () => {
+            process.env.CRAWLEE_MEMORY_MBYTES = '1024';
+            const config = new Configuration({ memoryMbytes: 2048 });
+
+            expect(config.get('memoryMbytes')).toBe(2048);
+        });
+
+        test('integer option: persistStateIntervalMillis', () => {
+            process.env.CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS = '30000';
+            const config = new Configuration({ persistStateIntervalMillis: 90000 });
+
+            expect(config.get('persistStateIntervalMillis')).toBe(90000);
+        });
+    });
+
+    describe('env vars take precedence over defaults', () => {
+        test('env var overrides default headless value', () => {
+            process.env.CRAWLEE_HEADLESS = 'false';
+            const config = new Configuration();
+
+            expect(config.get('headless')).toBe(false);
+        });
+
+        test('env var overrides default persistStateIntervalMillis', () => {
+            process.env.CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS = '120000';
+            const config = new Configuration();
+
+            expect(config.get('persistStateIntervalMillis')).toBe(120000);
+        });
+    });
+
+    describe('defaults are used when no other value is provided', () => {
+        test('uses default headless value', () => {
+            delete process.env.CRAWLEE_HEADLESS;
+            const config = new Configuration();
+
+            expect(config.get('headless')).toBe(true);
+        });
+
+        test('uses default persistStateIntervalMillis', () => {
+            delete process.env.CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS;
+            const config = new Configuration();
+
+            expect(config.get('persistStateIntervalMillis')).toBe(60_000);
+        });
+
+        test('uses default defaultDatasetId', () => {
+            delete process.env.CRAWLEE_DEFAULT_DATASET_ID;
+            const config = new Configuration();
+
+            expect(config.get('defaultDatasetId')).toBe('default');
+        });
+    });
+
+    describe('env vars are used when constructor option is not provided', () => {
+        test('uses env var when constructor does not specify the option', () => {
+            process.env.CRAWLEE_HEADLESS = 'false';
+            // Constructor provides a different option, not headless
+            const config = new Configuration({ persistStateIntervalMillis: 90000 });
+
+            // headless should come from env var since not in constructor
+            expect(config.get('headless')).toBe(false);
+            // persistStateIntervalMillis should come from constructor
+            expect(config.get('persistStateIntervalMillis')).toBe(90000);
+        });
+    });
+
+    describe('crawlee.json integration', () => {
+        test('constructor options override crawlee.json', () => {
+            writeFileSync(crawleeJsonPath, JSON.stringify({ headless: true, persistStateIntervalMillis: 30000 }));
+            createdCrawleeJson = true;
+
+            const config = new Configuration({ headless: false });
+
+            expect(config.get('headless')).toBe(false);
+            // persistStateIntervalMillis not in constructor, should come from crawlee.json
+            expect(config.get('persistStateIntervalMillis')).toBe(30000);
+        });
+
+        test('env vars override crawlee.json when constructor option not provided', () => {
+            writeFileSync(crawleeJsonPath, JSON.stringify({ headless: true }));
+            createdCrawleeJson = true;
+            process.env.CRAWLEE_HEADLESS = 'false';
+
+            const config = new Configuration();
+
+            expect(config.get('headless')).toBe(false);
+        });
+
+        test('crawlee.json values are used when no env var or constructor option', () => {
+            writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 45000 }));
+            createdCrawleeJson = true;
+            delete process.env.CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS;
+
+            const config = new Configuration();
+
+            expect(config.get('persistStateIntervalMillis')).toBe(45000);
+        });
+
+        test('full priority chain: constructor > env > crawlee.json > defaults', () => {
+            writeFileSync(crawleeJsonPath, JSON.stringify({
+                headless: true,
+                persistStateIntervalMillis: 45000,
+                defaultDatasetId: 'json-dataset',
+                inputKey: 'JSON_INPUT',
+            }));
+            createdCrawleeJson = true;
+
+            process.env.CRAWLEE_HEADLESS = 'false';
+            process.env.CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS = '30000';
+            delete process.env.CRAWLEE_DEFAULT_DATASET_ID;
+            delete process.env.CRAWLEE_INPUT_KEY;
+            delete process.env.CRAWLEE_PURGE_ON_START;
+
+            const config = new Configuration({
+                headless: true, // Should win over env var 'false'
+            });
+
+            // constructor wins over env var
+            expect(config.get('headless')).toBe(true);
+            // env var wins over crawlee.json (no constructor option for this)
+            expect(config.get('persistStateIntervalMillis')).toBe(30000);
+            // crawlee.json wins over default (no constructor or env var)
+            expect(config.get('defaultDatasetId')).toBe('json-dataset');
+            expect(config.get('inputKey')).toBe('JSON_INPUT');
+            // default is used (no constructor, env var, or crawlee.json)
+            expect(config.get('purgeOnStart')).toBe(true);
+        });
+    });
+
+    describe('edge cases', () => {
+        test('explicitly setting option to false overrides env var true', () => {
+            process.env.CRAWLEE_PURGE_ON_START = 'true';
+            const config = new Configuration({ purgeOnStart: false });
+
+            expect(config.get('purgeOnStart')).toBe(false);
+        });
+
+        test('explicitly setting option to 0 overrides env var', () => {
+            process.env.CRAWLEE_MEMORY_MBYTES = '1024';
+            const config = new Configuration({ memoryMbytes: 0 });
+
+            expect(config.get('memoryMbytes')).toBe(0);
+        });
+
+        test('explicitly setting option to empty string overrides env var', () => {
+            process.env.CRAWLEE_DEFAULT_DATASET_ID = 'env-dataset';
+            const config = new Configuration({ defaultDatasetId: '' });
+
+            expect(config.get('defaultDatasetId')).toBe('');
+        });
+    });
+});