diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ef4ec5d..21569e104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [patch] + +> Development of this release was supported by [Reset Tech](https://www.reset.tech). + +### Fixed + +- Fix browser and request language with full DOM fetcher + ## 10.3.0 - 2026-01-12 > Development of this release was made on a volunteer basis by [@Ndpnt](http://github.com/ndpnt). diff --git a/src/archivist/fetcher/fullDomFetcher.js b/src/archivist/fetcher/fullDomFetcher.js index 85858238c..ee05e3925 100644 --- a/src/archivist/fetcher/fullDomFetcher.js +++ b/src/archivist/fetcher/fullDomFetcher.js @@ -3,13 +3,14 @@ import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { resolveProxyConfiguration, extractProxyCredentials } from './proxyUtils.js'; -puppeteer.use(stealthPlugin()); - let browser; export default async function fetch(url, cssSelectors, config) { + puppeteer.use(stealthPlugin({ locale: config.language })); + let context; let page; + let client; let response; const selectors = [].concat(cssSelectors); @@ -25,6 +26,14 @@ export default async function fetch(url, cssSelectors, config) { await page.setDefaultNavigationTimeout(config.navigationTimeout); await page.setExtraHTTPHeaders({ 'Accept-Language': config.language }); + // Use CDP to ensure the browser language is set correctly (most reliable method, see https://zirkelc.dev/posts/puppeteer-language-experiment) + client = await page.createCDPSession(); + + await client.send('Network.setUserAgentOverride', { + userAgent: await browser.userAgent(), + acceptLanguage: config.language, + }); + if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) { await page.authenticate(browser.proxyCredentials); } @@ -73,6 +82,9 @@ export default async function fetch(url, cssSelectors, config) { } throw new Error(error.message); } finally { + if (client) { + await client.detach(); + } if (page) { await page.close(); }