From a166c7eb64be8f7a9dbf39db8dd0a9a4fdece87a Mon Sep 17 00:00:00 2001 From: Nikhil Sonti Date: Thu, 26 Feb 2026 19:23:14 -0800 Subject: [PATCH] feat: add zip archive support to download pipeline for bun-based server The new bun-based server uploads zip archives to R2 containing resources/bin/bun, resources/index.js, and resources/index.js.map. This updates the build pipeline to download and extract these zips instead of individual server binaries. - Add zip archive download + extraction support to download.py - Add {server_version} placeholder substitution in R2 keys - Update download_resources.yaml for zip archive downloads - Update copy_resources.yaml to copy extracted resource directories - Add BROWSEROS_SERVER_VERSION file for version tracking - Update validate_resources.py patch for new resource layout - Add resources/server/ to .gitignore Co-Authored-By: Claude Opus 4.6 --- .gitignore | 1 + .../build/config/copy_resources.yaml | 43 +++++----- .../build/config/download_resources.yaml | 54 +++++++----- .../build/modules/storage/download.py | 82 ++++++++++++++++--- .../browseros/server/validate_resources.py | 3 +- .../resources/BROWSEROS_SERVER_VERSION | 1 + 6 files changed, 131 insertions(+), 53 deletions(-) create mode 100644 packages/browseros/resources/BROWSEROS_SERVER_VERSION diff --git a/.gitignore b/.gitignore index ed429270b..f8d2b6333 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ gclient.json .env .grove/ **/resources/binaries/ +**/resources/server/ diff --git a/packages/browseros/build/config/copy_resources.yaml b/packages/browseros/build/config/copy_resources.yaml index d725d738c..72508cfbc 100644 --- a/packages/browseros/build/config/copy_resources.yaml +++ b/packages/browseros/build/config/copy_resources.yaml @@ -88,38 +88,39 @@ copy_operations: destination: "chrome/app/theme/default_200_percent/chromium" type: "directory" - # BrowserOS Server Binary - Platform & Architecture specific - - name: "BrowserOS Server Binary - macOS ARM64" - source: "resources/binaries/browseros_server/browseros-server-darwin-arm64" - destination: "chrome/browser/browseros/server/resources/bin/browseros_server" - type: "file" + # BrowserOS Server Resources - Platform & Architecture specific + # Copies extracted server zip contents (bin/bun, index.js, index.js.map) + - name: "BrowserOS Server Resources - macOS ARM64" + source: "resources/server/darwin-arm64/resources" + destination: "chrome/browser/browseros/server/resources" + type: "directory" os: ["macos"] arch: ["arm64"] - - name: "BrowserOS Server Binary - macOS x64" - source: "resources/binaries/browseros_server/browseros-server-darwin-x64" - destination: "chrome/browser/browseros/server/resources/bin/browseros_server" - type: "file" + - name: "BrowserOS Server Resources - macOS x64" + source: "resources/server/darwin-x64/resources" + destination: "chrome/browser/browseros/server/resources" + type: "directory" os: ["macos"] arch: ["x64"] - - name: "BrowserOS Server Binary - Linux ARM64" - source: "resources/binaries/browseros_server/browseros-server-linux-arm64" - destination: "chrome/browser/browseros/server/resources/bin/browseros_server" - type: "file" + - name: "BrowserOS Server Resources - Linux ARM64" + source: "resources/server/linux-arm64/resources" + destination: "chrome/browser/browseros/server/resources" + type: "directory" os: ["linux"] arch: ["arm64"] - - name: "BrowserOS Server Binary - Linux x64" - source: "resources/binaries/browseros_server/browseros-server-linux-x64" - destination: "chrome/browser/browseros/server/resources/bin/browseros_server" - type: "file" + - name: "BrowserOS Server Resources - Linux x64" + source: "resources/server/linux-x64/resources" + destination: "chrome/browser/browseros/server/resources" + type: "directory" os: ["linux"] arch: ["x64"] - - name: "BrowserOS Server Binary - Windows x64" - source: "resources/binaries/browseros_server/browseros-server-windows-x64.exe" - destination: "chrome/browser/browseros/server/resources/bin/browseros_server.exe" - type: "file" + - name: "BrowserOS Server Resources - Windows x64" + source: "resources/server/windows-x64/resources" + destination: "chrome/browser/browseros/server/resources" + type: "directory" os: ["windows"] arch: ["x64"] diff --git a/packages/browseros/build/config/download_resources.yaml b/packages/browseros/build/config/download_resources.yaml index d4f09a24b..0a09d5819 100644 --- a/packages/browseros/build/config/download_resources.yaml +++ b/packages/browseros/build/config/download_resources.yaml @@ -17,48 +17,64 @@ # - Use 'executable' field to mark files that need executable permissions # - Set to true for binaries (permissions are not preserved during download) # +# Archive Support: +# - Use 'archive: "zip"' to download and extract a zip archive +# - 'destination' becomes the extraction directory (cleared before extraction) +# - Use 'executable_paths' to set executable permissions on extracted files +# - Use '{server_version}' in r2_key to substitute the version from +# resources/BROWSEROS_SERVER_VERSION +# # R2 Path Structure: -# binaries/browseros-server/{filename} +# server/v{version}/browseros-server-{platform}.zip # # Example: -# - name: "My Binary - macOS ARM64" -# r2_key: "binaries/my-binary/my-binary-darwin-arm64" -# destination: "resources/binaries/my_binary/my-binary-darwin-arm64" +# - name: "My Archive - macOS ARM64" +# r2_key: "server/v{server_version}/my-archive-darwin-arm64.zip" +# destination: "resources/server/darwin-arm64" +# archive: "zip" +# executable_paths: ["resources/bin/my_binary"] # os: ["macos"] # arch: ["arm64"] download_operations: - # BrowserOS Server Binary - Platform & Architecture specific + # BrowserOS Server Resources - Platform & Architecture specific + # Downloads zip archives containing: resources/bin/bun, resources/index.js, resources/index.js.map - name: "BrowserOS Server - macOS ARM64" - r2_key: "binaries/browseros-server/browseros-server-darwin-arm64" - destination: "resources/binaries/browseros_server/browseros-server-darwin-arm64" + r2_key: "server/v{server_version}/browseros-server-darwin-arm64.zip" + destination: "resources/server/darwin-arm64" + archive: "zip" + executable_paths: ["resources/bin/bun"] os: ["macos"] arch: ["arm64"] - executable: true - name: "BrowserOS Server - macOS x64" - r2_key: "binaries/browseros-server/browseros-server-darwin-x64" - destination: "resources/binaries/browseros_server/browseros-server-darwin-x64" + r2_key: "server/v{server_version}/browseros-server-darwin-x64.zip" + destination: "resources/server/darwin-x64" + archive: "zip" + executable_paths: ["resources/bin/bun"] os: ["macos"] arch: ["x64"] - executable: true - name: "BrowserOS Server - Linux ARM64" - r2_key: "binaries/browseros-server/browseros-server-linux-arm64" - destination: "resources/binaries/browseros_server/browseros-server-linux-arm64" + r2_key: "server/v{server_version}/browseros-server-linux-arm64.zip" + destination: "resources/server/linux-arm64" + archive: "zip" + executable_paths: ["resources/bin/bun"] os: ["linux"] arch: ["arm64"] - executable: true - name: "BrowserOS Server - Linux x64" - r2_key: "binaries/browseros-server/browseros-server-linux-x64" - destination: "resources/binaries/browseros_server/browseros-server-linux-x64" + r2_key: "server/v{server_version}/browseros-server-linux-x64.zip" + destination: "resources/server/linux-x64" + archive: "zip" + executable_paths: ["resources/bin/bun"] os: ["linux"] arch: ["x64"] - executable: true - name: "BrowserOS Server - Windows x64" - r2_key: "binaries/browseros-server/browseros-server-windows-x64.exe" - destination: "resources/binaries/browseros_server/browseros-server-windows-x64.exe" + r2_key: "server/v{server_version}/browseros-server-windows-x64.zip" + destination: "resources/server/windows-x64" + archive: "zip" + executable_paths: ["resources/bin/bun.exe"] os: ["windows"] arch: ["x64"] diff --git a/packages/browseros/build/modules/storage/download.py b/packages/browseros/build/modules/storage/download.py index 019463095..98f823fd3 100644 --- a/packages/browseros/build/modules/storage/download.py +++ b/packages/browseros/build/modules/storage/download.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 """Download module for fetching build resources from Cloudflare R2""" +import shutil import yaml +import zipfile from pathlib import Path from typing import List @@ -84,28 +86,84 @@ def execute(self, context: Context) -> None: for op in filtered_ops: name = op.get("name", "Unnamed") - r2_key = op["r2_key"] + r2_key = self._resolve_r2_key(op["r2_key"], context) destination = op["destination"] + archive = op.get("archive") dest_path = context.root_dir / destination log_info(f" {name}") - # Clear existing file (always re-download) - if dest_path.exists(): - dest_path.unlink() - log_info(f" Cleared existing: {dest_path.name}") + if archive == "zip": + self._download_and_extract_zip( + client, r2_key, dest_path, bucket, op + ) + else: + # Clear existing file (always re-download) + if dest_path.exists(): + dest_path.unlink() + log_info(f" Cleared existing: {dest_path.name}") - # Download from R2 - if not download_file_from_r2(client, r2_key, dest_path, bucket): - raise RuntimeError(f"Failed to download: {name}") + if not download_file_from_r2(client, r2_key, dest_path, bucket): + raise RuntimeError(f"Failed to download: {name}") - # Set executable permissions if specified - if op.get("executable", False): - dest_path.chmod(dest_path.stat().st_mode | 0o755) - log_info(f" Set executable permissions") + if op.get("executable", False): + dest_path.chmod(dest_path.stat().st_mode | 0o755) + log_info(f" Set executable permissions") log_success(f"Downloaded {len(filtered_ops)} resource(s) from R2") + def _resolve_r2_key(self, r2_key: str, context: Context) -> str: + """Substitute {version} placeholders using version_file references.""" + if "{server_version}" not in r2_key: + return r2_key + + version_path = context.root_dir / "resources" / "BROWSEROS_SERVER_VERSION" + if not version_path.exists(): + raise RuntimeError( + f"Server version file not found: {version_path}" + ) + + version = version_path.read_text().strip() + return r2_key.replace("{server_version}", version) + + def _download_and_extract_zip( + self, + client, + r2_key: str, + extract_dir: Path, + bucket: str, + op: dict, + ) -> None: + """Download a zip archive from R2 and extract it.""" + # Clear existing extraction directory + if extract_dir.exists(): + shutil.rmtree(extract_dir) + log_info(f" Cleared existing: {extract_dir.name}") + + extract_dir.mkdir(parents=True, exist_ok=True) + + # Download to a temporary zip file + tmp_zip = extract_dir / "_download.zip" + if not download_file_from_r2(client, r2_key, tmp_zip, bucket): + raise RuntimeError(f"Failed to download: {op.get('name', r2_key)}") + + # Extract + log_info(f" Extracting archive...") + with zipfile.ZipFile(tmp_zip, "r") as zf: + zf.extractall(extract_dir) + + tmp_zip.unlink() + + # Set executable permissions on specified paths + executable_paths = op.get("executable_paths", []) + for rel_path in executable_paths: + exe_path = extract_dir / rel_path + if exe_path.exists(): + exe_path.chmod(exe_path.stat().st_mode | 0o755) + log_info(f" Set executable: {rel_path}") + + log_info(f" Extracted to: {extract_dir}") + def _filter_operations( self, operations: List[dict], diff --git a/packages/browseros/chromium_patches/chrome/browser/browseros/server/validate_resources.py b/packages/browseros/chromium_patches/chrome/browser/browseros/server/validate_resources.py index 29f0199fc..93c5c2045 100644 --- a/packages/browseros/chromium_patches/chrome/browser/browseros/server/validate_resources.py +++ b/packages/browseros/chromium_patches/chrome/browser/browseros/server/validate_resources.py @@ -20,7 +20,8 @@ +# Required resources that must exist in the resources/ directory +# Add more resources as needed - paths are relative to resources/ +REQUIRED_RESOURCES = [ -+ "bin/browseros_server", ++ "bin/bun", ++ "index.js", +] + +script_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/packages/browseros/resources/BROWSEROS_SERVER_VERSION b/packages/browseros/resources/BROWSEROS_SERVER_VERSION new file mode 100644 index 000000000..a758e3a84 --- /dev/null +++ b/packages/browseros/resources/BROWSEROS_SERVER_VERSION @@ -0,0 +1 @@ +0.0.57