feat(routeFromHar): add interceptAPIRequests option

docs/src/api/class-browsercontext.md

@@ -1307,6 +1307,12 @@ When set to `minimal`, only record information necessary for routing from HAR. T
 
 Optional setting to control resource content management. If `attach` is specified, resources are persisted as separate files or entries in the ZIP archive. If `embed` is specified, content is stored inline the HAR file.
 
+### option: BrowserContext.routeFromHAR.interceptAPIRequests
+* since: v1.62
+- `interceptAPIRequests` <[boolean]>
+
+If set to `true`, requests made via [APIRequestContext] (such as [`property: BrowserContext.request`] or [`property: Page.request`]) are also served from the HAR file. By default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for backward compatibility.
+
 
 ## async method: BrowserContext.routeWebSocket
 * since: v1.48

packages/isomorphic/protocolMetainfo.ts

@@ -89,6 +89,8 @@ export const methodMetainfo = new Map<string, MethodMetainfo>([
   ['BrowserContext.setGeolocation', { title: 'Set geolocation', group: 'configuration', }],
   ['BrowserContext.setHTTPCredentials', { title: 'Set HTTP credentials', group: 'configuration', }],
   ['BrowserContext.setNetworkInterceptionPatterns', { title: 'Route requests', group: 'route', }],
+  ['BrowserContext.harForAPIRequestsStart', { internal: true, }],
+  ['BrowserContext.harForAPIRequestsStop', { internal: true, }],
   ['BrowserContext.setWebSocketInterceptionPatterns', { title: 'Route WebSockets', group: 'route', }],
   ['BrowserContext.setOffline', { title: 'Set offline mode', }],
   ['BrowserContext.storageState', { title: 'Get storage state', group: 'configuration', }],

packages/playwright-client/types/types.d.ts

@@ -9402,6 +9402,15 @@ export interface BrowserContext {
    * @param options
    */
   routeFromHAR(har: string, options?: {
+    /**
+     * If set to `true`, requests made via [APIRequestContext](https://playwright.dev/docs/api/class-apirequestcontext)
+     * (such as [browserContext.request](https://playwright.dev/docs/api/class-browsercontext#browser-context-request) or
+     * [page.request](https://playwright.dev/docs/api/class-page#page-request)) are also served from the HAR file. By
+     * default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for
+     * backward compatibility.
+     */
+    interceptAPIRequests?: boolean;
+
     /**
      * - If set to 'abort' any request not found in the HAR file will be aborted.
      * - If set to 'fallback' falls through to the next route handler in the handler chain.

packages/playwright-core/src/client/browserContext.ts

@@ -385,7 +385,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     await this._updateWebSocketInterceptionPatterns({ title: 'Route WebSockets' });
   }
 
-  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full' } = {}): Promise<void> {
+  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full', interceptAPIRequests?: boolean } = {}): Promise<void> {
     const localUtils = this._connection.localUtils();
     if (!localUtils)
       throw new Error('Route from har is not supported in thin clients');
@@ -396,6 +396,8 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     const harRouter = await HarRouter.create(localUtils, har, options.notFound || 'abort', { urlMatch: options.url });
     this._harRouters.push(harRouter);
     await harRouter.addContextRoute(this);
+    if (options.interceptAPIRequests)
+      await harRouter.addAPIRequestRoute(this, har);
   }
 
   private _disposeHarRouters() {

packages/playwright-core/src/client/harRouter.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { isRegExp, isString } from '@isomorphic/rtti';
+
 import type { BrowserContext } from './browserContext';
 import type { LocalUtils } from './localUtils';
 import type { Route } from './network';
@@ -27,6 +29,7 @@ export class HarRouter {
   private _harId: string;
   private _notFoundAction: HarNotFoundAction;
   private _options: { urlMatch?: URLMatch; baseURL?: string; };
+  private _apiRequestRegistrations: { context: BrowserContext, registrationId: string }[] = [];
 
   static async create(localUtils: LocalUtils, file: string, notFoundAction: HarNotFoundAction, options: { urlMatch?: URLMatch }): Promise<HarRouter> {
     const { harId, error } = await localUtils.harOpen({ file });
@@ -115,11 +118,26 @@ export class HarRouter {
     await page.route(this._options.urlMatch || '**/*', route => this._handle(route));
   }
 
+  async addAPIRequestRoute(context: BrowserContext, har: string) {
+    const urlMatch = this._options.urlMatch;
+    const { registrationId } = await context._channel.harForAPIRequestsStart({
+      har,
+      urlGlob: isString(urlMatch) ? urlMatch : undefined,
+      urlRegexSource: isRegExp(urlMatch) ? urlMatch.source : undefined,
+      urlRegexFlags: isRegExp(urlMatch) ? urlMatch.flags : undefined,
+      notFound: this._notFoundAction,
+    });
+    this._apiRequestRegistrations.push({ context, registrationId });
+  }
+
   async [Symbol.asyncDispose]() {
     await this.dispose();
   }
 
   dispose() {
     this._localUtils.harClose({ harId: this._harId }).catch(() => {});
+    for (const { context, registrationId } of this._apiRequestRegistrations)
+      context._channel.harForAPIRequestsStop({ registrationId }).catch(() => {});
+    this._apiRequestRegistrations = [];
   }
 }

packages/playwright-core/src/protocol/validator.ts

@@ -830,6 +830,20 @@ scheme.BrowserContextSetNetworkInterceptionPatternsParams = tObject({
   })),
 });
 scheme.BrowserContextSetNetworkInterceptionPatternsResult = tOptional(tObject({}));
+scheme.BrowserContextHarForAPIRequestsStartParams = tObject({
+  har: tString,
+  urlGlob: tOptional(tString),
+  urlRegexSource: tOptional(tString),
+  urlRegexFlags: tOptional(tString),
+  notFound: tEnum(['abort', 'fallback']),
+});
+scheme.BrowserContextHarForAPIRequestsStartResult = tObject({
+  registrationId: tString,
+});
+scheme.BrowserContextHarForAPIRequestsStopParams = tObject({
+  registrationId: tString,
+});
+scheme.BrowserContextHarForAPIRequestsStopResult = tOptional(tObject({}));
 scheme.BrowserContextSetWebSocketInterceptionPatternsParams = tObject({
   patterns: tArray(tObject({
     glob: tOptional(tString),

packages/playwright-core/src/server/browserContext.ts

@@ -39,12 +39,14 @@ import type { Browser, BrowserOptions } from './browser';
 import type { ConsoleMessage } from './console';
 import type { Download } from './download';
 import type * as frames from './frames';
+import type { HarBackend } from './harBackend';
 import type { PageError } from './page';
 import type { Progress } from './progress';
 import type { ClientCertificatesProxy } from './socksClientCertificatesInterceptor';
 import type { SerializedStorage } from '@injected/storageScript';
 import type * as types from './types';
 import type * as channels from '@protocol/channels';
+import type { URLMatch } from '@isomorphic/urlMatch';
 
 const BrowserContextEvent = {
   Console: 'console',
@@ -120,6 +122,7 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
   private _playwrightBindingExposed?: Promise<void>;
   readonly dialogManager: DialogManager;
   private _consoleApiExposed = false;
+  private _harForAPIRequests: HarForAPIRequestsRegistration[] = [];
 
   constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
     super(browser, 'browser-context');
@@ -738,8 +741,36 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
   async notifyRoutesInFlightAboutRemovedHandler(handler: network.RouteHandler): Promise<void> {
     await Promise.all([...this._routesInFlight].map(route => route.removeHandler(handler)));
   }
+
+  addHarForAPIRequests(options: { harBackend: HarBackend, urlMatch: URLMatch | undefined, notFound: 'abort' | 'fallback', baseURL: string | undefined }): { dispose: () => void } {
+    const registration: HarForAPIRequestsRegistration = {
+      harBackend: options.harBackend,
+      urlMatch: options.urlMatch,
+      notFound: options.notFound,
+      baseURL: options.baseURL,
+    };
+    this._harForAPIRequests.push(registration);
+    return {
+      dispose: () => {
+        const index = this._harForAPIRequests.indexOf(registration);
+        if (index !== -1)
+          this._harForAPIRequests.splice(index, 1);
+      },
+    };
+  }
+
+  harForAPIRequests(): readonly HarForAPIRequestsRegistration[] {
+    return this._harForAPIRequests;
+  }
 }
 
+export type HarForAPIRequestsRegistration = {
+  harBackend: HarBackend;
+  urlMatch: URLMatch | undefined;
+  notFound: 'abort' | 'fallback';
+  baseURL: string | undefined;
+};
+
 export function validateBrowserContextOptions(options: types.BrowserContextOptions, browserOptions: BrowserOptions) {
   if (options.noDefaultViewport && options.deviceScaleFactor !== undefined)
     throw new Error(`"deviceScaleFactor" option is not supported with null "viewport"`);

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -39,6 +39,7 @@ import { RecorderApp } from '../recorder/recorderApp';
 import { ElementHandleDispatcher } from './elementHandlerDispatcher';
 import { JSHandleDispatcher } from './jsHandleDispatcher';
 import { disposeAll } from '../disposable';
+import { openHarBackend } from '../localUtils';
 
 import type { ConsoleMessage } from '../console';
 import type { Dialog } from '../dialog';
@@ -61,6 +62,7 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
   private _requestInterceptor: RouteHandler;
   private _interceptionUrlMatchers: URLMatch[] = [];
   private _routeWebSocketInitScript: InitScript | undefined;
+  private _harForAPIRequestsRegistrations = new Map<string, { dispose: () => void }>();
 
   static from(parentScope: DispatcherScope, context: BrowserContext): BrowserContextDispatcher {
     const result = parentScope.connection.existingDispatcher<BrowserContextDispatcher>(context);
@@ -335,6 +337,37 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
       this._routeWebSocketInitScript = await WebSocketRouteDispatcher.install(progress, this.connection, this._context);
   }
 
+  async harForAPIRequestsStart(params: channels.BrowserContextHarForAPIRequestsStartParams, progress: Progress): Promise<channels.BrowserContextHarForAPIRequestsStartResult> {
+    const result = await openHarBackend(progress, params.har);
+    if ('error' in result)
+      throw new Error(result.error);
+    const urlMatch: URLMatch | undefined =
+      params.urlRegexSource !== undefined && params.urlRegexFlags !== undefined ? new RegExp(params.urlRegexSource, params.urlRegexFlags) :
+        params.urlGlob !== undefined ? params.urlGlob : undefined;
+    const registrationId = createGuid();
+    const registration = this._context.addHarForAPIRequests({
+      harBackend: result.harBackend,
+      urlMatch,
+      notFound: params.notFound,
+      baseURL: this._context._options.baseURL,
+    });
+    this._harForAPIRequestsRegistrations.set(registrationId, {
+      dispose: () => {
+        registration.dispose();
+        result.harBackend.dispose();
+      },
+    });
+    return { registrationId };
+  }
+
+  async harForAPIRequestsStop(params: channels.BrowserContextHarForAPIRequestsStopParams, progress: Progress): Promise<void> {
+    const entry = this._harForAPIRequestsRegistrations.get(params.registrationId);
+    if (!entry)
+      return;
+    this._harForAPIRequestsRegistrations.delete(params.registrationId);
+    entry.dispose();
+  }
+
   async storageState(params: channels.BrowserContextStorageStateParams, progress: Progress): Promise<channels.BrowserContextStorageStateResult> {
     return await this._context.storageState(progress, params.indexedDB);
   }
@@ -446,6 +479,13 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
     this._context.dialogManager.removeDialogHandler(this._dialogHandler);
     this._interceptionUrlMatchers = [];
     this._context.removeRequestInterceptor(this._requestInterceptor).catch(() => {});
+    for (const entry of this._harForAPIRequestsRegistrations.values()) {
+      try {
+        entry.dispose();
+      } catch {
+      }
+    }
+    this._harForAPIRequestsRegistrations.clear();
     disposeAll(this._disposables).catch(() => {});
     if (this._routeWebSocketInitScript)
       WebSocketRouteDispatcher.uninstall(this.connection, this._context, this._routeWebSocketInitScript).catch(() => {});

packages/playwright-core/src/server/fetch.ts

@@ -23,7 +23,7 @@ import * as zlib from 'zlib';
 import { createGuid } from '@utils/crypto';
 import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent, timingForSocket } from '@utils/happyEyeballs';
 import { assert } from '@isomorphic/assert';
-import { constructURLBasedOnBaseURL } from '@isomorphic/urlMatch';
+import { constructURLBasedOnBaseURL, urlMatches } from '@isomorphic/urlMatch';
 import { eventsHelper } from '@utils/eventsHelper';
 import { monotonicTime } from '@isomorphic/time';
 import { createProxyAgent } from '@utils/network';
@@ -150,6 +150,10 @@ export abstract class APIRequestContext extends SdkObject {
   abstract addCookies(cookies: channels.NetworkCookie[]): Promise<void>;
   abstract cookies(progress: Progress, url: URL): Promise<channels.NetworkCookie[]>;
 
+  protected async _lookupInHar(progress: Progress, url: URL, method: string, headers: HeadersObject, postData: Buffer | undefined): Promise<SendRequestResult | undefined> {
+    return undefined;
+  }
+
   protected _disposeImpl() {
     this._disposed = true;
     APIRequestContext.allInstances.delete(this);
@@ -225,7 +229,14 @@ export abstract class APIRequestContext extends SdkObject {
     const postData = serializePostData(params, headers);
     if (postData)
       setHeader(headers, 'content-length', String(postData.byteLength));
-    const { body, log, response } = await this._sendRequestWithRetries(progress, requestUrl, options, postData, params.maxRetries);
+    const harResponse = await this._lookupInHar(progress, requestUrl, method, headers, postData);
+    let body: Buffer;
+    let log: string[];
+    let response: Omit<channels.APIResponse, 'fetchUid'>;
+    if (harResponse)
+      ({ body, log, response } = harResponse);
+    else
+      ({ body, log, response } = await this._sendRequestWithRetries(progress, requestUrl, options, postData, params.maxRetries));
     const failOnStatusCode = params.failOnStatusCode !== undefined ? params.failOnStatusCode : !!defaults.failOnStatusCode;
     if (failOnStatusCode && (response.status < 200 || response.status >= 400)) {
       let responseText = '';
@@ -682,6 +693,47 @@ export class BrowserContextAPIRequestContext extends APIRequestContext {
   override async storageState(progress: Progress, indexedDB?: boolean): Promise<channels.APIRequestContextStorageStateResult> {
     return this._context.storageState(progress, indexedDB);
   }
+
+  protected override async _lookupInHar(progress: Progress, url: URL, method: string, headers: HeadersObject, postData: Buffer | undefined): Promise<SendRequestResult | undefined> {
+    const registrations = this._context.harForAPIRequests();
+    if (!registrations.length)
+      return undefined;
+    const urlString = url.toString();
+    const log: string[] = [];
+    log.push(`→ ${method} ${urlString}`);
+    const headersArray: HeadersArray = Object.entries(headers).map(([name, value]) => ({ name, value }));
+    for (const registration of registrations) {
+      if (!urlMatches(registration.baseURL, urlString, registration.urlMatch))
+        continue;
+      const lookupResult = await progress.race(registration.harBackend.lookup(urlString, method, headersArray, postData, false, { apiRequestOnly: true }));
+      if (lookupResult.action === 'error') {
+        log.push(`HAR: ${lookupResult.message ?? 'lookup failed'}`);
+        continue;
+      }
+      if (lookupResult.action === 'noentry') {
+        if (registration.notFound === 'abort')
+          throw new Error(`Request "${method} ${urlString}" was not found in the HAR file`);
+        continue;
+      }
+      if (lookupResult.action === 'redirect') {
+        // Not expected for non-navigation API requests, but treat as fulfill miss.
+        log.push(`HAR: ignoring redirect entry for ${urlString}`);
+        continue;
+      }
+      log.push(`← ${lookupResult.status ?? 0} (from HAR)`);
+      return {
+        body: lookupResult.body ?? Buffer.from(''),
+        log,
+        response: {
+          url: urlString,
+          status: lookupResult.status ?? 0,
+          statusText: '',
+          headers: lookupResult.headers ?? [],
+        },
+      };
+    }
+    return undefined;
+  }
 }
 
 

packages/playwright-core/src/server/harBackend.ts

@@ -39,7 +39,7 @@ export class HarBackend {
     this._zipFile = zipFile;
   }
 
-  async lookup(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, isNavigationRequest: boolean): Promise<{
+  async lookup(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, isNavigationRequest: boolean, options: { apiRequestOnly?: boolean } = {}): Promise<{
     action: 'error' | 'redirect' | 'fulfill' | 'noentry',
     message?: string,
     redirectURL?: string,
@@ -49,7 +49,7 @@ export class HarBackend {
   }> {
     let entry;
     try {
-      entry = await this._harFindResponse(url, method, headers, postData);
+      entry = await this._harFindResponse(url, method, headers, postData, options);
     } catch (e) {
       return { action: 'error', message: 'HAR error: ' + e.message };
     }
@@ -93,14 +93,16 @@ export class HarBackend {
     return buffer;
   }
 
-  private async _harFindResponse(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined): Promise<har.Entry | undefined> {
+  private async _harFindResponse(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, options: { apiRequestOnly?: boolean } = {}): Promise<har.Entry | undefined> {
     const harLog = this._harFile.log;
     const visited = new Set<har.Entry>();
     while (true) {
       const entries: har.Entry[] = [];
       for (const candidate of harLog.entries) {
         if (candidate.request.url !== url || candidate.request.method !== method)
           continue;
+        if (options.apiRequestOnly && !candidate._apiRequest)
+          continue;
         if (method === 'POST' && postData && candidate.request.postData) {
           const buffer = await this._loadContent(candidate.request.postData);
           if (!buffer.equals(postData)) {