From d87f175298d0f58175d400cef5f29e2ccfe0c559 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 8 May 2026 23:55:47 -0700 Subject: [PATCH 1/4] feat(webfetch): route through fetch-use when BROWSER_USE_API_KEY is set Adds @browser-use/bcode-browser/fetch-use Effect service that POSTs to https://fetch.browser-use.com/fetch with the X-Browser-Use-API-Key header. The webfetch tool consults fetchUse.enabled and swaps the HTTP call when the key is present, preserving the existing native HttpClient + cloudflare-retry path otherwise. Opt-out: BCODE_NO_FETCH_USE=1. Decisions ref: decisions.md S3.3 / ROADMAP B1+B2. No webfetch schema changes (session_id / output_format / proxy_country deferred until evals show demand). B3 retired by Phase H. --- packages/bcode-browser/src/fetch-use.ts | 92 ++++++++++++++ packages/bcode-browser/test/fetch-use.test.ts | 40 ++++++ packages/opencode/src/tool/registry.ts | 3 + packages/opencode/src/tool/webfetch.ts | 114 ++++++++++-------- packages/opencode/test/session/prompt.test.ts | 2 + .../test/session/snapshot-tool-race.test.ts | 2 + packages/opencode/test/tool/registry.test.ts | 2 + packages/opencode/test/tool/webfetch.test.ts | 5 +- 8 files changed, 211 insertions(+), 49 deletions(-) create mode 100644 packages/bcode-browser/src/fetch-use.ts create mode 100644 packages/bcode-browser/test/fetch-use.test.ts diff --git a/packages/bcode-browser/src/fetch-use.ts b/packages/bcode-browser/src/fetch-use.ts new file mode 100644 index 000000000..004ffe613 --- /dev/null +++ b/packages/bcode-browser/src/fetch-use.ts @@ -0,0 +1,92 @@ +// fetch-use — Effect service that proxies HTTP requests through Browser Use's +// fetch-use cloud (Chrome JA4 fingerprint, HTTP/2 header order, session-based +// cookie persistence). See `memory/browsercode/fetch_use_reference.md` for +// the API shape; decisions.md §3.3 + ROADMAP B1 for the rationale. +// +// The layer is always constructible. `enabled` reflects whether +// BROWSER_USE_API_KEY is set and the user hasn't opted out via +// BCODE_NO_FETCH_USE=1. Consumers (webfetch.ts) check `enabled` and fall back +// to native HttpClient when false. + +import { Context, Effect, Layer } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" + +const ENDPOINT = "https://fetch.browser-use.com/fetch" +const DEFAULT_TIMEOUT_MS = 30_000 + +// Mirrors the Go FetchResponse type at +// github.com/browser-use/fetch-use/internal/types/types.go. `headers` is +// http.Header — `map[string][]string` over the wire — so each value is an +// array of strings, not a single string. +interface FetchUseRaw { + status_code: number + headers?: Record + body?: string + body_base64?: string + is_binary?: boolean + error?: string +} + +export interface FetchOptions { + readonly timeoutMs?: number +} + +export interface FetchResult { + readonly body: ArrayBuffer + readonly contentType: string + readonly statusCode: number +} + +export interface Interface { + readonly enabled: boolean + readonly fetch: (url: string, opts?: FetchOptions) => Effect.Effect +} + +export class Service extends Context.Service()("@browser-use/FetchUse") {} + +const headerValue = (h: Record | undefined, key: string): string => { + if (!h) return "" + for (const [k, v] of Object.entries(h)) { + if (k.toLowerCase() === key.toLowerCase()) return v[0] ?? "" + } + return "" +} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const apiKey = process.env.BROWSER_USE_API_KEY ?? "" + const enabled = apiKey.length > 0 && process.env.BCODE_NO_FETCH_USE !== "1" + + const fetch = (url: string, opts?: FetchOptions) => + Effect.gen(function* () { + const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS + const request = yield* HttpClientRequest.post(ENDPOINT).pipe( + HttpClientRequest.setHeaders({ + "Content-Type": "application/json", + "X-Browser-Use-API-Key": apiKey, + }), + HttpClientRequest.bodyJson({ url, timeout_ms: timeoutMs }), + ) + const response = yield* HttpClient.filterStatusOk(http).execute(request) + const data = (yield* response.json) as unknown as FetchUseRaw + if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`)) + + const body = + data.is_binary && data.body_base64 + ? new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer + : new TextEncoder().encode(data.body ?? "").buffer + + return { + body: body as ArrayBuffer, + contentType: headerValue(data.headers, "Content-Type"), + statusCode: data.status_code, + } + }).pipe(Effect.mapError((e) => (e instanceof Error ? e : new Error(String(e))))) + + return Service.of({ enabled, fetch }) + }), +) + +export * as FetchUse from "./fetch-use" diff --git a/packages/bcode-browser/test/fetch-use.test.ts b/packages/bcode-browser/test/fetch-use.test.ts new file mode 100644 index 000000000..916c1485b --- /dev/null +++ b/packages/bcode-browser/test/fetch-use.test.ts @@ -0,0 +1,40 @@ +// FetchUse smoke tests. +// +// Unit: layer is constructible, `enabled` reflects env vars correctly. +// Live: when BROWSER_USE_API_KEY is set, end-to-end POST to fetch.browser-use.com +// returns body bytes + content-type. Skipped without the key. + +import { expect, test } from "bun:test" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "../src/fetch-use" + +const haveKey = !!process.env.BROWSER_USE_API_KEY + +test("layer constructs and exposes `enabled`", async () => { + const enabled = await Effect.gen(function* () { + const svc = yield* FetchUse.Service + return svc.enabled + }).pipe( + Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), + Effect.runPromise, + ) + expect(typeof enabled).toBe("boolean") + expect(enabled).toBe(haveKey && process.env.BCODE_NO_FETCH_USE !== "1") +}) + +test.skipIf(!haveKey)("live: fetches httpbin and returns body + content-type", async () => { + const result = await Effect.gen(function* () { + const svc = yield* FetchUse.Service + return yield* svc.fetch("https://httpbin.org/get") + }).pipe( + Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), + Effect.runPromise, + ) + + expect(result.statusCode).toBe(200) + expect(result.contentType).toContain("application/json") + const text = new TextDecoder().decode(result.body) + const data = JSON.parse(text) + expect(data.url).toBe("https://httpbin.org/get") +}) diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index eb1d0391e..d7e354089 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -33,6 +33,7 @@ import path from "path" import { pathToFileURL } from "url" import { Effect, Layer, Context } from "effect" import { FetchHttpClient, HttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { Ripgrep } from "../file/ripgrep" @@ -85,6 +86,7 @@ export const layer: Layer.Layer< | AppFileSystem.Service | Bus.Service | HttpClient.HttpClient + | FetchUse.Service | ChildProcessSpawner | Ripgrep.Service | Format.Service @@ -349,6 +351,7 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Instruction.defaultLayer), Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Bus.layer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(Format.defaultLayer), Layer.provide(CrossSpawnSpawner.defaultLayer), diff --git a/packages/opencode/src/tool/webfetch.ts b/packages/opencode/src/tool/webfetch.ts index 65b718c3f..819e4d264 100644 --- a/packages/opencode/src/tool/webfetch.ts +++ b/packages/opencode/src/tool/webfetch.ts @@ -1,5 +1,6 @@ import { Effect, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import * as Tool from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" @@ -24,6 +25,7 @@ export const WebFetchTool = Tool.define( Effect.gen(function* () { const http = yield* HttpClient.HttpClient const httpOk = HttpClient.filterStatusOk(http) + const fetchUse = yield* FetchUse.Service return { description: DESCRIPTION, @@ -47,61 +49,77 @@ export const WebFetchTool = Tool.define( const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT) - // Build Accept header based on requested format with q parameters for fallbacks - let acceptHeader = "*/*" - switch (params.format) { - case "markdown": - acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" - break - case "text": - acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" - break - case "html": - acceptHeader = - "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" - break - default: - acceptHeader = - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" - } - const headers = { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", - Accept: acceptHeader, - "Accept-Language": "en-US,en;q=0.9", - } + // BrowserCode: when fetch-use is enabled (BROWSER_USE_API_KEY set, + // BCODE_NO_FETCH_USE != "1"), proxy through it for Chrome JA4 + // fingerprinting + HTTP/2 header order. Falls back to native + // HttpClient with cloudflare-retry when disabled. + const { arrayBuffer, contentType } = yield* (fetchUse.enabled + ? fetchUse + .fetch(params.url, { timeoutMs: timeout }) + .pipe(Effect.map((r) => ({ arrayBuffer: r.body, contentType: r.contentType }))) + : Effect.gen(function* () { + // Build Accept header based on requested format with q parameters for fallbacks + let acceptHeader = "*/*" + switch (params.format) { + case "markdown": + acceptHeader = + "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" + break + case "text": + acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" + break + case "html": + acceptHeader = + "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" + break + default: + acceptHeader = + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" + } + const headers = { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + Accept: acceptHeader, + "Accept-Language": "en-US,en;q=0.9", + } - const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers)) - - // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) - const response = yield* httpOk.execute(request).pipe( - Effect.catchIf( - (err) => - err.reason._tag === "StatusCodeError" && - err.reason.response.status === 403 && - err.reason.response.headers["cf-mitigated"] === "challenge", - () => - httpOk.execute( - HttpClientRequest.get(params.url).pipe( - HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }), + const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers)) + + // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) + const response = yield* httpOk.execute(request).pipe( + Effect.catchIf( + (err) => + err.reason._tag === "StatusCodeError" && + err.reason.response.status === 403 && + err.reason.response.headers["cf-mitigated"] === "challenge", + () => + httpOk.execute( + HttpClientRequest.get(params.url).pipe( + HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }), + ), + ), ), - ), - ), - Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }), - ) - - // Check content length - const contentLength = response.headers["content-length"] - if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)") - } + Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }), + ) + + // Check content length + const contentLength = response.headers["content-length"] + if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") + } + + const arrayBuffer = yield* response.arrayBuffer + if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") + } + + return { arrayBuffer, contentType: response.headers["content-type"] || "" } + })) - const arrayBuffer = yield* response.arrayBuffer if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { throw new Error("Response too large (exceeds 5MB limit)") } - const contentType = response.headers["content-type"] || "" const mime = contentType.split(";")[0]?.trim().toLowerCase() || "" const title = `${params.url} (${contentType})` diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index c5170f346..7156345bd 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -39,6 +39,7 @@ import { Shell } from "../../src/shell/shell" import { Snapshot } from "../../src/snapshot" import { ToolRegistry } from "@/tool/registry" import { Truncate } from "@/tool/truncate" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import * as Log from "@opencode-ai/core/util/log" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import * as Database from "../../src/storage/db" @@ -176,6 +177,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(CrossSpawnSpawner.defaultLayer), Layer.provide(Ripgrep.defaultLayer), diff --git a/packages/opencode/test/session/snapshot-tool-race.test.ts b/packages/opencode/test/session/snapshot-tool-race.test.ts index ab5a3ab7e..c615fbd08 100644 --- a/packages/opencode/test/session/snapshot-tool-race.test.ts +++ b/packages/opencode/test/session/snapshot-tool-race.test.ts @@ -50,6 +50,7 @@ import { SessionRunState } from "../../src/session/run-state" import { SessionStatus } from "../../src/session/status" import { Snapshot } from "../../src/snapshot" import { ToolRegistry } from "@/tool/registry" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Truncate } from "@/tool/truncate" import { AppFileSystem } from "@opencode-ai/core/filesystem" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" @@ -126,6 +127,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(CrossSpawnSpawner.defaultLayer), Layer.provide(Ripgrep.defaultLayer), diff --git a/packages/opencode/test/tool/registry.test.ts b/packages/opencode/test/tool/registry.test.ts index c33981ddf..d5448e44d 100644 --- a/packages/opencode/test/tool/registry.test.ts +++ b/packages/opencode/test/tool/registry.test.ts @@ -19,6 +19,7 @@ import { LSP } from "@/lsp/lsp" import { Instruction } from "@/session/instruction" import { Bus } from "@/bus" import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Format } from "@/format" import { Ripgrep } from "@/file/ripgrep" import * as Truncate from "@/tool/truncate" @@ -42,6 +43,7 @@ const registryLayer = ToolRegistry.layer.pipe( Layer.provide(Instruction.defaultLayer), Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Bus.layer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(Format.defaultLayer), Layer.provide(node), diff --git a/packages/opencode/test/tool/webfetch.test.ts b/packages/opencode/test/tool/webfetch.test.ts index 6c7f6aba7..491c6f346 100644 --- a/packages/opencode/test/tool/webfetch.test.ts +++ b/packages/opencode/test/tool/webfetch.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test" import path from "path" import { Effect, Layer } from "effect" import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Agent } from "../../src/agent/agent" import { Truncate } from "@/tool/truncate" import { Instance } from "../../src/project/instance" @@ -31,7 +32,9 @@ function exec(args: { url: string; format: "text" | "markdown" | "html" }) { return WebFetchTool.pipe( Effect.flatMap((info) => info.init()), Effect.flatMap((tool) => tool.execute(args, ctx)), - Effect.provide(Layer.mergeAll(FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer)), + Effect.provide( + Layer.mergeAll(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer)), FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer), + ), Effect.runPromise, ) } From 3cae8f4e9f63ed1d3aad5aad14d87cb1c92925fa Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 9 May 2026 01:00:29 -0700 Subject: [PATCH 2/4] fix(fetch-use): switch opt-out to experimental.fetch_use config; tighten service - Opt-out is now experimental.fetch_use=false in opencode.json (was BCODE_NO_FETCH_USE env var). One field added to the existing experimental config struct. - FetchUse service tightened from ~95 to ~55 LOC: inline interface, single object Service.of, dropped statusCode from FetchResult (unused). Behavior unchanged. - webfetch.ts opt-out check moves from service-internal env read to call-site config check. FetchUse.enabled now reflects only key presence. - Tests updated; live + native paths verified. --- packages/bcode-browser/src/fetch-use.ts | 97 ++++++------------- packages/bcode-browser/test/fetch-use.test.ts | 34 +++---- packages/opencode/src/config/config.ts | 4 + packages/opencode/src/tool/webfetch.ts | 11 ++- packages/opencode/test/tool/webfetch.test.ts | 9 +- 5 files changed, 61 insertions(+), 94 deletions(-) diff --git a/packages/bcode-browser/src/fetch-use.ts b/packages/bcode-browser/src/fetch-use.ts index 004ffe613..097a1de32 100644 --- a/packages/bcode-browser/src/fetch-use.ts +++ b/packages/bcode-browser/src/fetch-use.ts @@ -1,23 +1,18 @@ -// fetch-use — Effect service that proxies HTTP requests through Browser Use's -// fetch-use cloud (Chrome JA4 fingerprint, HTTP/2 header order, session-based -// cookie persistence). See `memory/browsercode/fetch_use_reference.md` for -// the API shape; decisions.md §3.3 + ROADMAP B1 for the rationale. -// -// The layer is always constructible. `enabled` reflects whether -// BROWSER_USE_API_KEY is set and the user hasn't opted out via -// BCODE_NO_FETCH_USE=1. Consumers (webfetch.ts) check `enabled` and fall back -// to native HttpClient when false. +// FetchUse — Effect service that proxies HTTP through Browser Use's fetch-use +// cloud (Chrome JA4, HTTP/2 header order, session cookies). Decisions §3.3. +// `enabled` is true when BROWSER_USE_API_KEY is set; webfetch.ts combines +// this with the user's `experimental.fetch_use` opencode.json setting. import { Context, Effect, Layer } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" const ENDPOINT = "https://fetch.browser-use.com/fetch" -const DEFAULT_TIMEOUT_MS = 30_000 -// Mirrors the Go FetchResponse type at -// github.com/browser-use/fetch-use/internal/types/types.go. `headers` is -// http.Header — `map[string][]string` over the wire — so each value is an -// array of strings, not a single string. +export interface FetchResult { + readonly body: ArrayBuffer + readonly contentType: string +} + interface FetchUseRaw { status_code: number headers?: Record @@ -27,65 +22,35 @@ interface FetchUseRaw { error?: string } -export interface FetchOptions { - readonly timeoutMs?: number -} - -export interface FetchResult { - readonly body: ArrayBuffer - readonly contentType: string - readonly statusCode: number -} - -export interface Interface { +export class Service extends Context.Service Effect.Effect -} - -export class Service extends Context.Service()("@browser-use/FetchUse") {} - -const headerValue = (h: Record | undefined, key: string): string => { - if (!h) return "" - for (const [k, v] of Object.entries(h)) { - if (k.toLowerCase() === key.toLowerCase()) return v[0] ?? "" - } - return "" -} + readonly fetch: (url: string, opts: { timeoutMs: number }) => Effect.Effect +}>()("@browser-use/FetchUse") {} export const layer = Layer.effect( Service, Effect.gen(function* () { const http = yield* HttpClient.HttpClient const apiKey = process.env.BROWSER_USE_API_KEY ?? "" - const enabled = apiKey.length > 0 && process.env.BCODE_NO_FETCH_USE !== "1" - - const fetch = (url: string, opts?: FetchOptions) => - Effect.gen(function* () { - const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS - const request = yield* HttpClientRequest.post(ENDPOINT).pipe( - HttpClientRequest.setHeaders({ - "Content-Type": "application/json", - "X-Browser-Use-API-Key": apiKey, - }), - HttpClientRequest.bodyJson({ url, timeout_ms: timeoutMs }), - ) - const response = yield* HttpClient.filterStatusOk(http).execute(request) - const data = (yield* response.json) as unknown as FetchUseRaw - if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`)) - - const body = - data.is_binary && data.body_base64 - ? new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer - : new TextEncoder().encode(data.body ?? "").buffer - - return { - body: body as ArrayBuffer, - contentType: headerValue(data.headers, "Content-Type"), - statusCode: data.status_code, - } - }).pipe(Effect.mapError((e) => (e instanceof Error ? e : new Error(String(e))))) - - return Service.of({ enabled, fetch }) + return Service.of({ + enabled: apiKey.length > 0, + fetch: (url, { timeoutMs }) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.post(ENDPOINT).pipe( + HttpClientRequest.setHeaders({ "Content-Type": "application/json", "X-Browser-Use-API-Key": apiKey }), + HttpClientRequest.bodyJson({ url, timeout_ms: timeoutMs }), + ) + const response = yield* HttpClient.filterStatusOk(http).execute(request) + const data = (yield* response.json) as unknown as FetchUseRaw + if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`)) + const body = data.is_binary && data.body_base64 + ? (new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer as ArrayBuffer) + : (new TextEncoder().encode(data.body ?? "").buffer as ArrayBuffer) + const ct = + Object.entries(data.headers ?? {}).find(([k]) => k.toLowerCase() === "content-type")?.[1]?.[0] ?? "" + return { body, contentType: ct } + }).pipe(Effect.mapError((e) => (e instanceof Error ? e : new Error(String(e))))), + }) }), ) diff --git a/packages/bcode-browser/test/fetch-use.test.ts b/packages/bcode-browser/test/fetch-use.test.ts index 916c1485b..34f7c1148 100644 --- a/packages/bcode-browser/test/fetch-use.test.ts +++ b/packages/bcode-browser/test/fetch-use.test.ts @@ -1,8 +1,10 @@ // FetchUse smoke tests. // -// Unit: layer is constructible, `enabled` reflects env vars correctly. -// Live: when BROWSER_USE_API_KEY is set, end-to-end POST to fetch.browser-use.com -// returns body bytes + content-type. Skipped without the key. +// Unit: layer is constructible, `enabled` reflects BROWSER_USE_API_KEY presence. +// Live: when the key is set, end-to-end POST to fetch.browser-use.com returns +// body bytes + content-type. Skipped without the key. Config-based +// opt-out (experimental.fetch_use=false) is enforced in webfetch.ts, +// not here. import { expect, test } from "bun:test" import { Effect, Layer } from "effect" @@ -11,30 +13,18 @@ import { FetchUse } from "../src/fetch-use" const haveKey = !!process.env.BROWSER_USE_API_KEY -test("layer constructs and exposes `enabled`", async () => { +test("layer constructs and exposes `enabled` reflecting env", async () => { const enabled = await Effect.gen(function* () { - const svc = yield* FetchUse.Service - return svc.enabled - }).pipe( - Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), - Effect.runPromise, - ) - expect(typeof enabled).toBe("boolean") - expect(enabled).toBe(haveKey && process.env.BCODE_NO_FETCH_USE !== "1") + return (yield* FetchUse.Service).enabled + }).pipe(Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), Effect.runPromise) + expect(enabled).toBe(haveKey) }) test.skipIf(!haveKey)("live: fetches httpbin and returns body + content-type", async () => { const result = await Effect.gen(function* () { - const svc = yield* FetchUse.Service - return yield* svc.fetch("https://httpbin.org/get") - }).pipe( - Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), - Effect.runPromise, - ) + return yield* (yield* FetchUse.Service).fetch("https://httpbin.org/get", { timeoutMs: 30_000 }) + }).pipe(Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), Effect.runPromise) - expect(result.statusCode).toBe(200) expect(result.contentType).toContain("application/json") - const text = new TextDecoder().decode(result.body) - const data = JSON.parse(text) - expect(data.url).toBe("https://httpbin.org/get") + expect(JSON.parse(new TextDecoder().decode(result.body)).url).toBe("https://httpbin.org/get") }) diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 3bd070047..c3acc7c2a 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -263,6 +263,10 @@ export const Info = Schema.Struct({ mcp_timeout: Schema.optional(PositiveInt).annotate({ description: "Timeout in milliseconds for model context protocol (MCP) requests", }), + fetch_use: Schema.optional(Schema.Boolean).annotate({ + description: + "Route webfetch through the Browser Use fetch-use proxy when BROWSER_USE_API_KEY is set. Defaults to true; set false to opt out (still costs but uses native HttpClient instead).", + }), }), ), }) diff --git a/packages/opencode/src/tool/webfetch.ts b/packages/opencode/src/tool/webfetch.ts index 819e4d264..82a98de43 100644 --- a/packages/opencode/src/tool/webfetch.ts +++ b/packages/opencode/src/tool/webfetch.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { FetchUse } from "@browser-use/bcode-browser/fetch-use" +import { Config } from "@/config/config" import * as Tool from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" @@ -26,6 +27,7 @@ export const WebFetchTool = Tool.define( const http = yield* HttpClient.HttpClient const httpOk = HttpClient.filterStatusOk(http) const fetchUse = yield* FetchUse.Service + const config = yield* Config.Service return { description: DESCRIPTION, @@ -49,11 +51,10 @@ export const WebFetchTool = Tool.define( const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT) - // BrowserCode: when fetch-use is enabled (BROWSER_USE_API_KEY set, - // BCODE_NO_FETCH_USE != "1"), proxy through it for Chrome JA4 - // fingerprinting + HTTP/2 header order. Falls back to native - // HttpClient with cloudflare-retry when disabled. - const { arrayBuffer, contentType } = yield* (fetchUse.enabled + // BrowserCode: route through fetch-use when BROWSER_USE_API_KEY is + // set and the user hasn't opted out via experimental.fetch_use=false. + const useFu = fetchUse.enabled && (yield* config.get()).experimental?.fetch_use !== false + const { arrayBuffer, contentType } = yield* (useFu ? fetchUse .fetch(params.url, { timeoutMs: timeout }) .pipe(Effect.map((r) => ({ arrayBuffer: r.body, contentType: r.contentType }))) diff --git a/packages/opencode/test/tool/webfetch.test.ts b/packages/opencode/test/tool/webfetch.test.ts index 491c6f346..2d8e6c389 100644 --- a/packages/opencode/test/tool/webfetch.test.ts +++ b/packages/opencode/test/tool/webfetch.test.ts @@ -4,6 +4,7 @@ import { Effect, Layer } from "effect" import { FetchHttpClient } from "effect/unstable/http" import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Agent } from "../../src/agent/agent" +import { Config } from "@/config/config" import { Truncate } from "@/tool/truncate" import { Instance } from "../../src/project/instance" import { WithInstance } from "../../src/project/with-instance" @@ -33,7 +34,13 @@ function exec(args: { url: string; format: "text" | "markdown" | "html" }) { Effect.flatMap((info) => info.init()), Effect.flatMap((tool) => tool.execute(args, ctx)), Effect.provide( - Layer.mergeAll(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer)), FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer), + Layer.mergeAll( + FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer)), + FetchHttpClient.layer, + Config.defaultLayer, + Truncate.defaultLayer, + Agent.defaultLayer, + ), ), Effect.runPromise, ) From 278dba63c8afc3b1ec10ca52725562736ff98263 Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 9 May 2026 14:15:37 -0700 Subject: [PATCH 3/4] docs(BROWSER.md): note webfetch enhancement when BROWSER_USE_API_KEY is set --- packages/bcode-browser/skills/BROWSER.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/bcode-browser/skills/BROWSER.md b/packages/bcode-browser/skills/BROWSER.md index 21cfc1c07..ee13cae03 100644 --- a/packages/bcode-browser/skills/BROWSER.md +++ b/packages/bcode-browser/skills/BROWSER.md @@ -98,6 +98,8 @@ console.log("liveUrl for the user to watch:", liveUrl) Requires `BROWSER_USE_API_KEY` in the environment (the user should have set this before launching bcode). If absent, tell the user to get a key at https://browser-use.com and `export BROWSER_USE_API_KEY=...`. +When `BROWSER_USE_API_KEY` is set, `webfetch` is automatically enhanced with `fetch-use` (Chrome TLS fingerprint + residential proxy + session cookies) — each request is free, but consumes a small amount of proxy bandwidth from the BU account. Disable in `opencode.json` with `experimental.fetch_use: false`. + ## Attaching to a target After `connect()`, attach to a page target before driving the browser: From 9394b918b6c1e95a8464c67f109c2df409a283de Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 9 May 2026 14:30:54 -0700 Subject: [PATCH 4/4] fix(fetch-use): fail on upstream HTTP error status to match native path fetch-use returns HTTP 200 with {status_code: N, body: ...} for any upstream response, so the existing filterStatusOk only catches errors talking to fetch.browser-use.com itself, not 4xx/5xx from the target URL. Native webfetch path uses filterStatusOk on the target directly and fails on non-2xx; fetch-use branch was silently surfacing 404/500 pages as successful content. Add explicit status_code >= 400 check to match. --- packages/bcode-browser/src/fetch-use.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/bcode-browser/src/fetch-use.ts b/packages/bcode-browser/src/fetch-use.ts index 097a1de32..83562b982 100644 --- a/packages/bcode-browser/src/fetch-use.ts +++ b/packages/bcode-browser/src/fetch-use.ts @@ -43,6 +43,8 @@ export const layer = Layer.effect( const response = yield* HttpClient.filterStatusOk(http).execute(request) const data = (yield* response.json) as unknown as FetchUseRaw if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`)) + // Mirror native path's filterStatusOk: surface upstream HTTP errors as failures. + if (data.status_code >= 400) return yield* Effect.fail(new Error(`fetch-use: HTTP ${data.status_code}`)) const body = data.is_binary && data.body_base64 ? (new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer as ArrayBuffer) : (new TextEncoder().encode(data.body ?? "").buffer as ArrayBuffer)