diff --git a/extensions/chromium-control-canvas/README.md b/extensions/chromium-control-canvas/README.md new file mode 100644 index 000000000..9fbe1849e --- /dev/null +++ b/extensions/chromium-control-canvas/README.md @@ -0,0 +1,82 @@ +# Chromium Control Canvas + +A GitHub Copilot canvas that drives a real **headful Chromium** window via Playwright. +The host app's built-in `browser` canvas is WebKit (WKWebView); this gives you actual +Chromium, controllable both from the panel UI and by the agent. + +The canvas panel is a control strip (URL bar, back/forward/reload, screenshot). A separate +Chromium window does the real rendering, because you can't embed Chromium inside a WebKit +iframe. + +## Files + +- `extension.mjs` — the extension: canvas declaration, Playwright launch, a loopback HTTP + server for the panel, and the agent actions. +- `index.html` — the control strip UI the panel renders. +- `package.json` — declares the `playwright` dependency and `"type": "module"`. +- `copilot-extension.json` — name/version metadata. + +## Prerequisites + +- **Node.js 18 or newer** (Playwright 1.60 requires `node >=18`; older versions hit a + cryptic engine error). The extension runs as a Node child process. +- The app's **canvas / UI-extensions experiment enabled**. Without it, the extension + loads but the canvas never appears in the panel. Enable it in the app's + Settings → Experiments. (This may not be available to all accounts.) + +## Install + +Drop this folder at `~/.copilot/extensions/chromium-control-canvas/` (user scope) or in a repo's +`.github/extensions/chromium-control-canvas/` (project scope), then install dependencies and the +Chromium binary from inside that folder: + +```sh +cd ~/.copilot/extensions/chromium-control-canvas +npm install # playwright is declared in package.json +npx playwright install chromium # downloads the browser, a few hundred MB +``` + +Reload extensions in the app, then open the `chromium-control-canvas` canvas. + +Note: copying the extension files only places the source. It does **not** run the +commands above or enable the experiment, so those steps are still required on first +setup. + +## Attach to your own Chrome + +By default the canvas launches the bundled Chromium with a persistent profile. To drive +a Chrome you already have running instead, start it with a debug port and pass `cdpUrl` +when opening the canvas: + +```sh +google-chrome --remote-debugging-port=9222 # then open the canvas with cdpUrl: http://localhost:9222 +``` + +In this mode the extension connects over CDP and never launches or kills your browser; +closing the canvas just disconnects. + +## Agent actions + +- `navigate { url }` — go to a URL or search query (blocklist-guarded). +- `back` / `forward` / `reload` — history navigation. +- `current_url` — current URL and page title. +- `snapshot` — structured list of visible interactive elements, each with a stable ref. +- `click { ref | selector }` — click an element by snapshot ref or CSS selector. +- `type { ref | selector, text, submit? }` — fill an input; optionally press Enter. +- `screenshot { fullPage? }` — save a PNG to `artifacts/` and return its path and size. + +## Notes + +- A persistent profile is stored in `profile/` so logins survive restarts. **Do not commit + or share `profile/`** — it contains real session cookies. +- Raw `evaluate` (arbitrary in-page JS) is intentionally omitted. +- `navigate` is checked against a blocklist, and a request interceptor also blocks + navigations to blocked hosts that happen via in-page redirects. The shipped + `BLOCKLIST` entries are illustrative examples, not real coverage — edit the list in + `extension.mjs` to fit your environment. +- The loopback control server requires a per-launch token (templated into the panel), + so other pages in your browser can't drive it. +- Typed text (e.g. passwords) is redacted in `audit.log`, and password field values are + excluded from snapshots. +- Generated at runtime and not part of the source: `node_modules/`, `profile/`, + `artifacts/`, `audit.log`. diff --git a/extensions/chromium-control-canvas/copilot-extension.json b/extensions/chromium-control-canvas/copilot-extension.json new file mode 100644 index 000000000..2c4915b99 --- /dev/null +++ b/extensions/chromium-control-canvas/copilot-extension.json @@ -0,0 +1,4 @@ +{ + "name": "chromium-control-canvas", + "version": 1 +} diff --git a/extensions/chromium-control-canvas/extension.mjs b/extensions/chromium-control-canvas/extension.mjs new file mode 100644 index 000000000..d54ed9d0d --- /dev/null +++ b/extensions/chromium-control-canvas/extension.mjs @@ -0,0 +1,630 @@ +// Extension: chromium-control-canvas +// Launches a real headful Chromium window via Playwright and uses the canvas +// panel as a control strip (URL bar, back, forward, reload, screenshot, +// snapshot/click/type). +// +// Why this shape: the host app renders canvases in a WebKit (WKWebView) webview, +// not Chromium. To get a true Chromium engine we run the browser as a separate +// headful window owned by this extension process and drive it with Playwright. +// The canvas iframe is only the control surface; it POSTs commands to a +// per-instance loopback HTTP server, which calls Playwright on the page. The +// same handlers are exposed as agent-callable canvas actions. +// +// Patterns borrowed from AndreaGriffiths11/claw-relay (grep "[claw-relay]" below +// for the exact spots): +// - persistent profile so logins survive restarts +// - optional connect-to-existing-Chrome over CDP instead of relaunching +// - a real action set (snapshot/click/type/screenshot), not just navigation +// - ref-based element resolution from a snapshot, or raw CSS selector +// - a site blocklist guard and a JSONL audit log +// Intentionally omitted: raw `evaluate` (arbitrary JS execution). + +import { randomUUID } from "node:crypto"; +import { appendFile, mkdir, readFile, readlink, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + CanvasError, + createCanvas, + joinSession, +} from "@github/copilot-sdk/extension"; +import { chromium } from "playwright"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const COPILOT_HOME = process.env.COPILOT_HOME || join(homedir(), ".copilot"); +const EXT_HOME = join(COPILOT_HOME, "extensions", "chromium-control-canvas"); +// [claw-relay] Persistent profile: cookies/logins survive canvas closes, +// reloads, and sessions, so a hand-login in the window sticks. +const PROFILE_DIR = join(EXT_HOME, "profile"); +const ARTIFACTS_DIR = join(EXT_HOME, "artifacts"); +const AUDIT_LOG = join(EXT_HOME, "audit.log"); + +// [claw-relay] Site blocklist guard. +// Sites the agent may never drive. Edit this list to taste. Patterns match the +// hostname; a leading "*." matches any subdomain. Navigation to a blocked host +// is refused before Chromium is told to go there. +const BLOCKLIST = [ + "*.bank.com", + "*.chase.com", + "*.paypal.com", + "accounts.google.com", +]; + +// One Chromium context + control-strip server per open canvas instance. +const instances = new Map(); // instanceId -> { context, browser, page, server, url, mode } + +// Per-launch secret, templated into index.html and required on every state route +// so cross-origin pages in the user's normal browser can't POST to this server. +const TOKEN = randomUUID(); + +let log = (..._args) => {}; + +function hostMatches(host, pattern) { + if (pattern.startsWith("*.")) { + const base = pattern.slice(2); + return host === base || host.endsWith(`.${base}`); + } + return host === pattern; +} + +function blockedReason(targetUrl) { + let host; + try { + host = new URL(targetUrl).hostname; + } catch (_) { + return null; + } + const hit = BLOCKLIST.find((p) => hostMatches(host, p)); + return hit ? `${host} is blocked by the Chromium Control Canvas blocklist (${hit})` : null; +} + +function normalizeUrl(input) { + const raw = String(input ?? "").trim(); + if (!raw) return "about:blank"; + if (raw === "about:blank") return raw; + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return raw; + // Local dev servers have no dot; send them to http, not search. + if (/^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?([/?#]|$)/i.test(raw)) return `http://${raw}`; + if (!/\s/.test(raw) && /\.[a-z]{2,}/i.test(raw)) return `https://${raw}`; + return `https://www.google.com/search?q=${encodeURIComponent(raw)}`; +} + +// [claw-relay] JSONL audit log: one line per action (panel or agent) for a +// reviewable trail of what drove the browser. +async function audit(entry) { + const line = JSON.stringify({ at: new Date().toISOString(), ...entry }); + await mkdir(EXT_HOME, { recursive: true }).catch(() => {}); + await appendFile(AUDIT_LOG, `${line}\n`).catch(() => {}); +} + +// Keep secrets out of the audit log: redact free-text fields (e.g. a typed +// password) while preserving the rest of the entry for the trail. +function redactInput(input) { + if (input && typeof input === "object" && "text" in input) { + return { ...input, text: "[redacted]" }; + } + return input; +} + +async function pageState(page) { + let url = "about:blank"; + let title = ""; + try { + url = page.url(); + } catch (_) {} + try { + title = await page.title(); + } catch (_) {} + return { url, title }; +} + +function getInstance(instanceId) { + const entry = instances.get(instanceId); + if (!entry) { + throw new CanvasError( + "no_instance", + "No open Chromium canvas for this instance. Open the canvas first.", + ); + } + return entry; +} + +// Return a usable page for the instance, recovering if the user closed the tab +// or window. Reuses an open tab, opens a new one in the live context, or (for +// persistent mode) relaunches the browser in place. +async function livePage(entry) { + if (entry.page && !entry.page.isClosed()) return entry.page; + try { + const open = entry.context?.pages().find((p) => !p.isClosed()); + entry.page = open || (await entry.context.newPage()); + return entry.page; + } catch (_) { + // Context/browser is gone below. + } + if (entry.mode === "persistent") { + // Memoize the relaunch so two concurrent panel requests after a crash + // don't race launchPersistentContext into the lock error we handle below. + if (!entry.relaunching) { + entry.relaunching = (async () => { + await mkdir(PROFILE_DIR, { recursive: true }); + await clearStaleLockIfDead(); + const context = await chromium.launchPersistentContext(PROFILE_DIR, PERSISTENT_OPTS); + await installGuards(context); + entry.context = context; + entry.page = context.pages()[0] || (await context.newPage()); + return entry.page; + })(); + entry.relaunching.then( + () => { + entry.relaunching = null; + }, + () => { + entry.relaunching = null; + }, + ); + } + return entry.relaunching; + } + throw new CanvasError( + "disconnected", + "The Chrome connection was lost. Close this panel and reopen the canvas.", + ); +} + +async function navigate(page, rawUrl) { + const url = normalizeUrl(rawUrl); + const reason = blockedReason(url); + if (reason) throw new CanvasError("site_blocked", reason); + await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); + return pageState(page); +} + +// [claw-relay] ref-based element resolution. Resolve an element target to a +// Playwright selector. `selector` wins over `ref` when both are given. `ref` +// values come from a prior snapshot and are stamped onto the DOM as +// data-cc-ref attributes. +function resolveTarget(input) { + if (input?.selector) return input.selector; + if (input?.ref) return `[data-cc-ref="${String(input.ref).replace(/"/g, "")}"]`; + throw new CanvasError("bad_target", "Provide a `ref` (from snapshot) or a `selector`."); +} + +// [claw-relay] Accessibility-style page snapshot: enumerate visible interactive +// elements and stamp each with a stable ref (e1, e2, ...) the agent can target. +const SNAPSHOT_FN = () => { + const sel = [ + "a[href]", + "button", + "input", + "textarea", + "select", + "[role=button]", + "[role=link]", + "[role=textbox]", + "[role=checkbox]", + "[onclick]", + '[contenteditable=""]', + '[contenteditable="true"]', + ].join(","); + const out = []; + let i = 0; + // Clear refs from a prior snapshot so a renumbered ref can't match a stale + // element that scrolled out of view. + for (const stamped of document.querySelectorAll("[data-cc-ref]")) { + stamped.removeAttribute("data-cc-ref"); + } + for (const el of document.querySelectorAll(sel)) { + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + const visible = + rect.width > 0 && + rect.height > 0 && + style.visibility !== "hidden" && + style.display !== "none"; + if (!visible) continue; + i += 1; + if (i > 200) break; + const ref = `e${i}`; + el.setAttribute("data-cc-ref", ref); + const name = ( + el.getAttribute("aria-label") || + el.getAttribute("placeholder") || + (el.type === "password" ? "" : el.value) || + el.innerText || + el.getAttribute("title") || + "" + ) + .trim() + .replace(/\s+/g, " ") + .slice(0, 80); + out.push({ + ref, + tag: el.tagName.toLowerCase(), + type: el.getAttribute("type") || el.getAttribute("role") || "", + name, + }); + } + return out; +}; + +async function snapshot(page) { + const elements = await page.evaluate(SNAPSHOT_FN); + return { ...(await pageState(page)), elements }; +} + +async function clickTarget(page, input) { + const selector = resolveTarget(input); + await page.click(selector, { timeout: 15000 }); + return pageState(page); +} + +async function typeTarget(page, input) { + const text = String(input?.text ?? ""); + const selector = resolveTarget(input); + const locator = page.locator(selector).first(); + await locator.fill(text, { timeout: 15000 }); + if (input?.submit) await locator.press("Enter").catch(() => {}); + return pageState(page); +} + +async function screenshot(page, opts = {}) { + await mkdir(ARTIFACTS_DIR, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const name = `shot-${stamp}.png`; + const buffer = await page.screenshot({ fullPage: !!opts.fullPage }); + await writeFile(join(ARTIFACTS_DIR, name), buffer); + return { name, path: join(ARTIFACTS_DIR, name), size: buffer.length }; +} + +function sendJson(res, status, body) { + res.writeHead(status, { "Content-Type": "application/json; charset=utf-8" }); + res.end(JSON.stringify(body)); +} + +function publicErrorMessage(err, fallback) { + return err instanceof CanvasError ? err.message : fallback; +} + +async function readBody(req) { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf-8"); +} + +function makeHandler(entry) { + return async function handleRequest(req, res) { + const { pathname } = new URL(req.url, "http://127.0.0.1"); + + if (req.method === "GET" && (pathname === "/" || pathname === "/index.html")) { + const html = (await readFile(join(__dirname, "index.html"), "utf-8")).replaceAll("__TOKEN__", TOKEN); + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(html); + return; + } + + if (req.method === "GET" && pathname.startsWith("/shot/")) { + const name = decodeURIComponent(pathname.slice("/shot/".length)); + if (!/^shot-[\w-]+\.png$/.test(name)) { + sendJson(res, 400, { error: "invalid name" }); + return; + } + const bytes = await readFile(join(ARTIFACTS_DIR, name)); + res.writeHead(200, { "Content-Type": "image/png", "Cache-Control": "no-store" }); + res.end(bytes); + return; + } + + // Every state route below requires the per-launch token templated into + // index.html. A cross-origin page can't read it, and the custom header + // also forces a CORS preflight this server never answers with allow. + if (req.headers["x-canvas-token"] !== TOKEN) { + sendJson(res, 403, { error: "forbidden" }); + return; + } + + const page = await livePage(entry); + + if (req.method === "GET" && pathname === "/state") { + sendJson(res, 200, { ...(await pageState(page)), mode: entry.mode }); + return; + } + + if (req.method === "POST" && pathname === "/navigate") { + const body = JSON.parse((await readBody(req)) || "{}"); + try { + const state = await navigate(page, body?.url); + await audit({ source: "panel", instanceId: entry.instanceId, action: "navigate", input: body?.url, url: state.url, ok: true }); + sendJson(res, 200, state); + } catch (err) { + sendJson(res, 200, { ...(await pageState(page)), error: publicErrorMessage(err, "Navigation failed.") }); + } + return; + } + + const simple = { "/back": "goBack", "/forward": "goForward", "/reload": "reload" }; + if (req.method === "POST" && simple[pathname]) { + await page[simple[pathname]]({ waitUntil: "domcontentloaded" }).catch(() => {}); + sendJson(res, 200, await pageState(page)); + return; + } + + if (req.method === "POST" && pathname === "/screenshot") { + const shot = await screenshot(page); + sendJson(res, 200, shot); + return; + } + + sendJson(res, 404, { error: "not found" }); + }; +} + +async function startServer(entry) { + const handler = makeHandler(entry); + const server = createServer((req, res) => { + handler(req, res).catch((err) => { + sendJson(res, 500, { error: publicErrorMessage(err, "Request failed.") }); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + return { server, url: `http://127.0.0.1:${port}/` }; +} + +const PERSISTENT_OPTS = { headless: false, viewport: null }; + +// [claw-relay] Enforce the blocklist on real navigations, not just explicit +// navigate() calls, so in-page redirects are caught too. Every request +// round-trips through Node, which is fine at agent/human browsing pace. +async function installGuards(context) { + await context.route("**/*", (route) => { + const req = route.request(); + if (req.isNavigationRequest() && blockedReason(req.url())) { + return route.abort("blockedbyclient"); + } + return route.continue(); + }); +} + +// Chromium writes a SingletonLock symlink (target: "-") into the +// profile while a window owns it. A reload or killed process can leave that +// lock behind even though no Chromium is running. If the referenced PID is +// dead, the lock is stale and safe to clear; if it's alive, a real window +// owns the profile and we must not touch it. +async function clearStaleLockIfDead() { + const lockPath = join(PROFILE_DIR, "SingletonLock"); + let target; + try { + target = await readlink(lockPath); + } catch (_) { + return false; // no lock present + } + const pid = Number(target.split("-").pop()); + if (Number.isFinite(pid) && pid > 0) { + try { + process.kill(pid, 0); // throws if the process is gone + return false; // alive -> a real window owns the profile + } catch (err) { + if (err?.code === "EPERM") return false; // exists, not ours -> leave it + } + } + for (const f of ["SingletonLock", "SingletonSocket", "SingletonCookie"]) { + await rm(join(PROFILE_DIR, f), { force: true }).catch(() => {}); + } + return true; +} + +async function openInstance(instanceId, input) { + const cdpUrl = input?.cdpUrl; + let context; + let browser = null; + let mode; + if (cdpUrl) { + // [claw-relay] Connect to an already-running Chrome over CDP (started with + // --remote-debugging-port) instead of launching our own window. + browser = await chromium.connectOverCDP(cdpUrl); + context = browser.contexts()[0] || (await browser.newContext()); + mode = "cdp"; + } else { + // One shared persistent profile can only back one live Chromium window. + // Refuse a second persistent instance with a readable message instead of + // a cryptic Playwright lock dump. + for (const e of instances.values()) { + if (e.mode === "persistent") { + throw new CanvasError( + "profile_busy", + "A Chromium canvas is already open. Close it before opening another — they share one logged-in profile.", + ); + } + } + // Persistent profile: same Chromium user-data-dir every time, so logins + // you complete by hand in the window survive across sessions. + await mkdir(PROFILE_DIR, { recursive: true }); + try { + context = await chromium.launchPersistentContext(PROFILE_DIR, PERSISTENT_OPTS); + } catch (err) { + if (!/existing browser session/i.test(String(err?.message))) throw err; + const cleared = await clearStaleLockIfDead(); + if (!cleared) { + throw new CanvasError( + "profile_busy", + "The Chromium profile is in use by another live window. Close that Chromium window first.", + ); + } + context = await chromium.launchPersistentContext(PROFILE_DIR, PERSISTENT_OPTS); + } + mode = "persistent"; + } + await installGuards(context); + const page = context.pages()[0] || (await context.newPage()); + const entry = { instanceId, context, browser, page, mode }; + await navigate(page, input?.url || "about:blank").catch(() => {}); + const { server, url } = await startServer(entry); + entry.server = server; + entry.url = url; + instances.set(instanceId, entry); + await audit({ instanceId, action: "open", mode, url: input?.url || "about:blank" }); + return entry; +} + +async function closeInstance(instanceId) { + const entry = instances.get(instanceId); + if (!entry) return; + instances.delete(instanceId); + await new Promise((resolve) => { + entry.server.closeAllConnections?.(); + entry.server.close(() => resolve()); + }).catch(() => {}); + if (entry.mode === "cdp") { + // Don't kill the user's own Chrome; just disconnect. + await entry.browser?.close().catch(() => {}); + } else { + await entry.context.close().catch(() => {}); + } + await audit({ instanceId, action: "close", mode: entry.mode }); +} + +// Wrap a canvas action handler so every agent-driven call is audited. +function action(name, description, run, inputSchema) { + return { + name, + description, + ...(inputSchema ? { inputSchema } : {}), + handler: async (ctx) => { + const entry = getInstance(ctx.instanceId); + try { + const page = await livePage(entry); + const result = await run(page, ctx.input, entry); + const state = result?.url ? result : await pageState(page); + await audit({ source: "agent", instanceId: ctx.instanceId, action: name, input: redactInput(ctx.input), url: state.url, ok: true }); + return result; + } catch (err) { + await audit({ source: "agent", instanceId: ctx.instanceId, action: name, input: redactInput(ctx.input), ok: false, error: publicErrorMessage(err, "Action failed.") }); + throw err; + } + }, + }; +} + +const session = await joinSession({ + canvases: [ + createCanvas({ + id: "chromium-control-canvas", + displayName: "Chromium Control Canvas", + description: + "Control canvas for a real headful Chromium window driven by Playwright.", + inputSchema: { + type: "object", + properties: { + url: { type: "string", description: "Optional URL to open on launch." }, + cdpUrl: { + type: "string", + description: + "Optional CDP endpoint (e.g. http://localhost:9222) to attach to an existing Chrome instead of launching one.", + }, + }, + }, + actions: [ + action( + "navigate", + "Navigate to a URL or search query (blocklist enforced).", + (page, input) => navigate(page, input?.url), + { + type: "object", + properties: { url: { type: "string", description: "URL (https assumed) or search query." } }, + required: ["url"], + }, + ), + action("back", "Go back in history.", async (page) => { + await page.goBack({ waitUntil: "domcontentloaded" }).catch(() => {}); + return pageState(page); + }), + action("forward", "Go forward in history.", async (page) => { + await page.goForward({ waitUntil: "domcontentloaded" }).catch(() => {}); + return pageState(page); + }), + action("reload", "Reload the current page.", async (page) => { + await page.reload({ waitUntil: "domcontentloaded" }).catch(() => {}); + return pageState(page); + }), + action("current_url", "Get the current URL and page title.", (page) => pageState(page)), + action( + "snapshot", + "List visible interactive elements with stable refs (e1, e2, ...) for click/type.", + (page) => snapshot(page), + ), + action( + "click", + "Click an element by `ref` (from snapshot) or CSS `selector`.", + (page, input) => clickTarget(page, input), + { + type: "object", + properties: { + ref: { type: "string", description: "Element ref from a snapshot, e.g. e3." }, + selector: { type: "string", description: "CSS selector (takes priority over ref)." }, + }, + }, + ), + action( + "type", + "Fill text into an input by `ref` or `selector`; set submit to press Enter.", + (page, input) => typeTarget(page, input), + { + type: "object", + properties: { + text: { type: "string", description: "Text to enter." }, + ref: { type: "string", description: "Element ref from a snapshot." }, + selector: { type: "string", description: "CSS selector (takes priority over ref)." }, + submit: { type: "boolean", description: "Press Enter after filling." }, + }, + required: ["text"], + }, + ), + action( + "screenshot", + "Capture a PNG of the page, saved under the extension artifacts dir.", + (page, input) => screenshot(page, { fullPage: input?.fullPage }), + { + type: "object", + properties: { fullPage: { type: "boolean", description: "Capture the full scrollable page." } }, + }, + ), + ], + open: async (ctx) => { + let entry = instances.get(ctx.instanceId); + if (!entry) { + entry = await openInstance(ctx.instanceId, ctx.input || {}); + log(`Launched Chromium (${entry.mode}) for instance ${ctx.instanceId}`, { + level: "info", + ephemeral: true, + }); + } + return { title: "Chromium Control Canvas", url: entry.url }; + }, + onClose: async (ctx) => { + await closeInstance(ctx.instanceId); + }, + }), + ], +}); + +log = (message, opts) => session.log(message, opts); + +// Close Chromium contexts on shutdown so reloading the extension doesn't orphan +// the window and leave a stale profile lock behind. The runtime sends SIGTERM +// (then SIGKILL after ~5s), so keep teardown fast. +let shuttingDown = false; +async function shutdown() { + if (shuttingDown) return; + shuttingDown = true; + await Promise.allSettled( + [...instances.keys()].map((id) => closeInstance(id)), + ); + process.exit(0); +} +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); diff --git a/extensions/chromium-control-canvas/index.html b/extensions/chromium-control-canvas/index.html new file mode 100644 index 000000000..f86b8c9a7 --- /dev/null +++ b/extensions/chromium-control-canvas/index.html @@ -0,0 +1,396 @@ + + + + + +Chromium + + + +
+ +
+
+
+
+ + + +
+
+ 🌐 + +
+ + +
+
+ … + about:blank +
+
+
+ +
+
+ +
+
+ Screenshot + +
+
+
+ This panel drives a separate Chromium window. Navigate above and + a snapshot of the page shows up here. Log in by hand in that window; the profile + persists. The agent can also snapshot, click, and + type. +
+
+
+ +
+ The preview updates after you navigate from this panel. Agent click and + type actions go straight to the page and bypass this panel, so the + preview can lag during agent work — focus the panel to resync. +
+
+ + + + diff --git a/extensions/chromium-control-canvas/package-lock.json b/extensions/chromium-control-canvas/package-lock.json new file mode 100644 index 000000000..dc195aba3 --- /dev/null +++ b/extensions/chromium-control-canvas/package-lock.json @@ -0,0 +1,252 @@ +{ + "name": "chromium-control-canvas", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "chromium-control-canvas", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@github/copilot-sdk": "latest", + "playwright": "^1.60.0" + } + }, + "node_modules/@github/copilot": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.61.tgz", + "integrity": "sha512-E4f7YXTL2uUZY/ypnfsUruAeSgrHx3AGYEbm5N0DrpzPqoNAZqV6kHEWM4vu+W/nGvydIfPxmOTqaMEhM8r0Uw==", + "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, + "bin": { + "copilot": "npm-loader.js" + }, + "optionalDependencies": { + "@github/copilot-darwin-arm64": "1.0.61", + "@github/copilot-darwin-x64": "1.0.61", + "@github/copilot-linux-arm64": "1.0.61", + "@github/copilot-linux-x64": "1.0.61", + "@github/copilot-linuxmusl-arm64": "1.0.61", + "@github/copilot-linuxmusl-x64": "1.0.61", + "@github/copilot-win32-arm64": "1.0.61", + "@github/copilot-win32-x64": "1.0.61" + } + }, + "node_modules/@github/copilot-darwin-arm64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.61.tgz", + "integrity": "sha512-10prvjHRXB0SD28NsIpzdNDgLquQYUwaH5Ev9KVdIWdBPAvlQsHmQ4JSCyD/UILc/nrrr02CKUgum+mZRKUKIg==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-arm64": "copilot" + } + }, + "node_modules/@github/copilot-darwin-x64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.61.tgz", + "integrity": "sha512-NXUjageJ3mxDfHtXGYu//XhJ+dhJFYObT4R3jeWgIHhd+4lX7FlC754nwlBP/ZuVhJ3ND22JK9sua9d2F3Cbwg==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-x64": "copilot" + } + }, + "node_modules/@github/copilot-linux-arm64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.61.tgz", + "integrity": "sha512-dwB2+QSMr622JkePeK56M7YWXsTT/DQzKfpDq8Lk2kmGU052RZAarRmt8gcNm4anofN7pMSrqc3YHj1TM84MFw==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linux-x64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.61.tgz", + "integrity": "sha512-q6n8R8oybvuCmmkP+43w809Wpud/wwRi/fFSZEYJagiNGmYJ00SDkrfJxHbZsAFMpaJC+oTswqzJHjRoZbO74w==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-x64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.61.tgz", + "integrity": "sha512-yWo7JXnZS11eJpm68E1RWKMR47EwzPKj3V7GX0EMTd8Fw0T2Aurk9wt9p3c9w0v02nTO1DqJhi68KVWJPdVqvA==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.61.tgz", + "integrity": "sha512-nHzx27Ac4B0fpD9CcmvyrGOBEMJ01CPRgVRP0yAl4wpU4cM2I6+9TPyfYThlWDqZqiUKGXC1ZRQ+B8cJREVGmA==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, + "node_modules/@github/copilot-sdk": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.1.tgz", + "integrity": "sha512-w6AaS0WqqTE/3iyUrZznvgCLQhsUF7ZmEVCneacuHCfOzlH0r6ww9WUmyA0zgqmXO75V0IYrkIcnFke/qJkkDg==", + "license": "MIT", + "dependencies": { + "@github/copilot": "^1.0.61", + "vscode-jsonrpc": "^8.2.1", + "zod": "^4.3.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@github/copilot-win32-arm64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.61.tgz", + "integrity": "sha512-k6knzI+K5HlZeJDS/yeJAfoYD4xcURWfuqunpTCyk1pDbIFxmrLSqR/TDi7KNlpsf883n5WqpnB06K5kysdHHQ==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-arm64": "copilot.exe" + } + }, + "node_modules/@github/copilot-win32-x64": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.61.tgz", + "integrity": "sha512-L6NZ6o73VZFHd7OoRaztV3Prh1PbW9HXqYsAx+XywNALQvE1u489WBUC1ggfYBW5MTBCf8mxSkYQdb3Am2omsw==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-x64": "copilot.exe" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz", + "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.60.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/extensions/chromium-control-canvas/package.json b/extensions/chromium-control-canvas/package.json new file mode 100644 index 000000000..f52d5aa67 --- /dev/null +++ b/extensions/chromium-control-canvas/package.json @@ -0,0 +1,14 @@ +{ + "name": "chromium-control-canvas", + "version": "1.0.0", + "description": "GitHub Copilot canvas that drives a real headful Chromium window via Playwright.", + "main": "extension.mjs", + "keywords": [], + "author": "Andrea Griffiths", + "license": "MIT", + "type": "module", + "dependencies": { + "@github/copilot-sdk": "latest", + "playwright": "^1.60.0" + } +}