From 5761353fc484c793658a4e570ffc41ca87548e8e Mon Sep 17 00:00:00 2001 From: LiHaohua Date: Tue, 30 Jun 2026 17:53:50 +0800 Subject: [PATCH 1/2] czdev: publish/unpublish via Release manifests instead of Git LFS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packages repo no longer stores .deb in git/LFS (LFS budget exhausted). Adapt the publish flow to the new manifest model: - publish: upload the .deb to a "czdev-buffer" Release on the push target (the contributor's own fork for non-maintainers — their free Release quota), then open a PR carrying only meta.json/screenshots/icon and a __.deb.release.json manifest (url + sha256 + size). No .deb, no LFS. Raises the size cap to the 2 GiB release-asset limit. - unpublish: read the manifest, download the .deb from its url to verify the Maintainer, and remove the manifest from git. - github_client: add ensure_release / upload_release_asset / find/delete asset. - Drop the git-lfs requirement. Co-authored-by: Cursor --- scripts/czdev/github_client.py | 47 +++++++++++++++++++ scripts/czdev/publish.py | 82 +++++++++++++++++++-------------- scripts/czdev/unpublish.py | 84 ++++++++++++---------------------- 3 files changed, 124 insertions(+), 89 deletions(-) diff --git a/scripts/czdev/github_client.py b/scripts/czdev/github_client.py index 4309e6d..bc22f04 100644 --- a/scripts/czdev/github_client.py +++ b/scripts/czdev/github_client.py @@ -2,11 +2,13 @@ import urllib.request import urllib.error +import urllib.parse import json import ssl from typing import Optional GITHUB_API = "https://api.github.com" +GITHUB_UPLOADS = "https://uploads.github.com" class Permission: @@ -131,6 +133,51 @@ def create_pull_request(self, owner: str, repo: str, title: str, body: str, head }) return PullRequestResponse(html_url=data["html_url"], number=data["number"]) + def ensure_release(self, owner: str, repo: str, tag: str, + name: Optional[str] = None, prerelease: bool = True) -> dict: + """Return the release for `tag`, creating it (prerelease) if missing.""" + try: + return self._get(f"/repos/{owner}/{repo}/releases/tags/{tag}") + except urllib.error.HTTPError as e: + if e.code != 404: + raise + return self._post(f"/repos/{owner}/{repo}/releases", body={ + "tag_name": tag, + "name": name or tag, + "prerelease": prerelease, + "body": "czdev upload buffer. Holds .deb assets referenced by package PRs.", + }) + + def find_release_asset(self, release: dict, name: str) -> Optional[dict]: + for asset in release.get("assets", []): + if asset.get("name") == name: + return asset + return None + + def delete_release_asset(self, owner: str, repo: str, asset_id: int) -> None: + self._request("DELETE", f"/repos/{owner}/{repo}/releases/assets/{asset_id}") + + def upload_release_asset(self, owner: str, repo: str, release: dict, + file_path: str, name: str) -> str: + """Upload `file_path` as a release asset, replacing any existing one. + + Returns the browser_download_url. + """ + existing = self.find_release_asset(release, name) + if existing: + self.delete_release_asset(owner, repo, existing["id"]) + release_id = release["id"] + url = f"{GITHUB_UPLOADS}/repos/{owner}/{repo}/releases/{release_id}/assets?name={urllib.parse.quote(name)}" + with open(file_path, "rb") as f: + data = f.read() + req = urllib.request.Request(url, data=data, method="POST") + req.add_header("Authorization", f"Bearer {self.token}") + req.add_header("User-Agent", "czdev/0.1") + req.add_header("Accept", "application/vnd.github+json") + req.add_header("Content-Type", "application/octet-stream") + resp = urllib.request.urlopen(req, context=self._ctx) + return json.loads(resp.read().decode())["browser_download_url"] + def get_file_content(self, owner: str, repo: str, path: str) -> bytes: url = f"{GITHUB_API}/repos/{owner}/{repo}/contents/{path}" req = urllib.request.Request(url, method="GET") diff --git a/scripts/czdev/publish.py b/scripts/czdev/publish.py index bf2df09..2acdda8 100644 --- a/scripts/czdev/publish.py +++ b/scripts/czdev/publish.py @@ -18,10 +18,13 @@ TARGET_OWNER = "CardputerZero" TARGET_REPO = "packages" +# Buffer release tag on the push target (contributor fork, or the official repo +# for maintainers) where the .deb is uploaded before review. +BUFFER_RELEASE_TAG = "czdev-buffer" def run(deb: Optional[str] = None): - check_git_lfs_installed() + check_git_installed() deb_path = resolve_deb(deb) print(f"Package: {deb_path}") @@ -68,8 +71,8 @@ def run(deb: Optional[str] = None): print(f" ✓ Version: {meta['version']}, Arch: {meta['architecture']}, Size: {size_mb:.1f} MB") print() - if file_size > 100 * 1024 * 1024: - print(f"ERROR: File too large. GitHub blob API limit is 100 MB. ({size_mb:.1f} MB)", file=sys.stderr) + if file_size > 2 * 1024 * 1024 * 1024: + print(f"ERROR: File too large. GitHub release asset limit is 2 GiB. ({size_mb:.1f} MB)", file=sys.stderr) sys.exit(1) # 5. Check version is newer than existing @@ -92,39 +95,58 @@ def run(deb: Optional[str] = None): branch = branch_name(meta) pr_head = f"{user.login}:{branch}" - # Upload via git init + fetch + lfs push + # Integrity + canonical asset name. GitHub sanitizes '~' to '.' in release + # asset names, so the on-release/manifest filename may differ from the + # Debian version string (which keeps '~'). file_bytes = Path(deb_path).read_bytes() sha256_hash = hashlib.sha256(file_bytes).hexdigest() - file_path_in_repo = f"pool/main/{meta['package']}/{meta['package']}_{meta['version']}_{meta['architecture']}.deb" + file_size = len(file_bytes) + deb_name = f"{meta['package']}_{meta['version']}_{meta['architecture']}.deb" + asset_name = deb_name.replace("~", ".") + manifest_name = f"{asset_name}.release.json" + manifest_path_in_repo = f"pool/main/{meta['package']}/{manifest_name}" branch = branch_name(meta) remote_url = f"git@github.com:{push_owner}/{push_repo}.git" - print(f"Uploading to {TARGET_OWNER}/{TARGET_REPO}...") + # 1) Upload the .deb to a buffer Release on the push target. For third-party + # contributors this is their own fork — it uses THEIR free Release storage + # and bandwidth, so the upstream project stores nothing until the PR is + # approved and CI promotes the binary into the official apt-pool release. + print(f" → Uploading .deb ({size_mb:.1f} MB) to {push_owner}/{push_repo} release '{BUFFER_RELEASE_TAG}'... ", + end="", flush=True) + release = gh.ensure_release(push_owner, push_repo, BUFFER_RELEASE_TAG, name="czdev upload buffer") + download_url = gh.upload_release_asset(push_owner, push_repo, release, deb_path, asset_name) + print("done") + + manifest = { + "filename": asset_name, + "url": download_url, + "sha256": sha256_hash, + "size": file_size, + "package": meta["package"], + "version": meta["version"], + "architecture": meta["architecture"], + } + + # 2) Commit only metadata (meta.json, screenshots, icon, manifest) to a PR + # branch — no .deb, no LFS. + print(f"Uploading metadata to {TARGET_OWNER}/{TARGET_REPO}...") tmp_dir = Path(tempfile.mkdtemp(prefix="czdev-publish-")) try: - # Init bare-minimum repo with LFS run_cmd_in(tmp_dir, ["git", "init"]) run_cmd_in(tmp_dir, ["git", "remote", "add", "origin", remote_url]) - run_cmd_in(tmp_dir, ["git", "lfs", "install", "--local"]) - run_cmd_in(tmp_dir, ["git", "config", - "lfs.https://github.com/CardputerZero/packages.git/info/lfs.locksverify", "false"]) - # Fetch only the tip of main print(" → git fetch (minimal)... ", end="", flush=True) run_cmd_in(tmp_dir, ["git", "fetch", "--depth=1", "--filter=blob:none", "origin", "main"]) print("done") - # Create branch from fetched main print(f" → Creating branch {branch}... ", end="", flush=True) run_cmd_in(tmp_dir, ["git", "checkout", "-b", branch, "origin/main"]) print("done") - # Copy deb into place dest_dir = tmp_dir / "pool" / "main" / meta["package"] dest_dir.mkdir(parents=True, exist_ok=True) - deb_dest_name = f"{meta['package']}_{meta['version']}_{meta['architecture']}.deb" - shutil.copy2(deb_path, dest_dir / deb_dest_name) # Copy screenshots if store_meta.get("screenshots"): @@ -143,19 +165,17 @@ def run(deb: Optional[str] = None): if icon_src.exists(): shutil.copy2(icon_src, dest_dir / icon_src.name) - # Generate meta.json - meta_json = json.dumps(store_meta, indent=2, ensure_ascii=False) - (dest_dir / "meta.json").write_text(meta_json) + # meta.json + release manifest (the binary stays in the buffer release) + (dest_dir / "meta.json").write_text(json.dumps(store_meta, indent=2, ensure_ascii=False)) + (dest_dir / manifest_name).write_text(json.dumps(manifest, indent=2) + "\n") - # Add and commit print(" → Creating commit... ", end="", flush=True) run_cmd_in(tmp_dir, ["git", "add", f"pool/main/{meta['package']}"]) run_cmd_in(tmp_dir, ["git", "commit", "-m", f"publish: {meta['package']} {meta['version']} ({meta['architecture']})"]) print("done") - # Push branch - print(f" → Uploading blob ({size_mb:.1f} MB)... ", end="", flush=True) + print(" → Pushing branch... ", end="", flush=True) run_cmd_in(tmp_dir, ["git", "push", "origin", branch]) print("done") @@ -173,8 +193,11 @@ def run(deb: Optional[str] = None): f"| Maintainer | {meta['maintainer']} |\n" f"| Size | {size_mb:.1f} MB |\n" f"| SHA-256 | `{sha256_hash}` |\n" - f"| File | `{file_path_in_repo}` |\n\n" - f"Submitted via `czdev publish`." + f"| Manifest | `{manifest_path_in_repo}` |\n" + f"| Binary | [{asset_name}]({download_url}) |\n\n" + f"Submitted via `czdev publish`. The `.deb` is hosted on the contributor's " + f"buffer release; CI verifies the sha256 and, on merge, promotes it into the " + f"official `apt-pool` release." ) print(" → Creating pull request... ", end="", flush=True) pr = gh.create_pull_request( @@ -357,24 +380,13 @@ def parse(v): return 0 -def check_git_lfs_installed(): +def check_git_installed(): try: subprocess.run(["git", "--version"], capture_output=True, check=True) except (subprocess.CalledProcessError, FileNotFoundError): print("git is not installed.\n Install: https://git-scm.com/downloads", file=sys.stderr) sys.exit(1) - try: - subprocess.run(["git", "lfs", "version"], capture_output=True, check=True) - except (subprocess.CalledProcessError, FileNotFoundError): - print("git-lfs is required for publishing.\n", file=sys.stderr) - print(" Install:", file=sys.stderr) - print(" macOS: brew install git-lfs", file=sys.stderr) - print(" Linux: sudo apt install git-lfs", file=sys.stderr) - print(" Windows: https://git-lfs.com", file=sys.stderr) - print("\n Then run: git lfs install", file=sys.stderr) - sys.exit(1) - def run_cmd_in(cwd: Path, cmd: list): result = subprocess.run(cmd, cwd=str(cwd), capture_output=True, text=True) diff --git a/scripts/czdev/unpublish.py b/scripts/czdev/unpublish.py index a09b154..1678173 100644 --- a/scripts/czdev/unpublish.py +++ b/scripts/czdev/unpublish.py @@ -1,10 +1,12 @@ """Unpublish (remove) a package from the CardputerZero app store — mirrors the Rust unpublish module.""" +import json import shutil import subprocess import sys import tempfile import time +import urllib.request from pathlib import Path from . import auth @@ -24,17 +26,36 @@ def run(package: str, version: str, arch: str = "arm64"): if user.email: all_emails.append(user.email) - file_path = f"pool/main/{package}/{package}_{version}_{arch}.deb" + # Packages are referenced by a manifest in git; the .deb itself lives in a + # Release. GitHub sanitizes '~' to '.' in release asset names. + asset_name = f"{package}_{version}_{arch}.deb".replace("~", ".") + file_path = f"pool/main/{package}/{asset_name}.release.json" - # Verify the file exists and belongs to this user print(f"Checking ownership of {package} {version}...") - # Use git sparse checkout + LFS to fetch just this one deb file + # Read the manifest from the repo, download the .deb it points at, and + # verify the Maintainer matches this user. tmp_dir = Path(tempfile.mkdtemp(prefix="czdev-unpublish-")) try: - deb_local = fetch_deb_via_git(tmp_dir, file_path) - if deb_local is None: - print("ERROR: package not found in repository", file=sys.stderr) + try: + manifest_raw = gh.get_file_content(TARGET_OWNER, TARGET_REPO, file_path) + except FileNotFoundError: + print("ERROR: package manifest not found in repository", file=sys.stderr) + sys.exit(1) + try: + manifest = json.loads(manifest_raw) + deb_url = manifest["url"] + except (json.JSONDecodeError, KeyError): + print("ERROR: invalid manifest (missing url)", file=sys.stderr) + sys.exit(1) + + deb_local = tmp_dir / asset_name + try: + req = urllib.request.Request(deb_url, headers={"User-Agent": "czdev/0.1"}) + with urllib.request.urlopen(req, timeout=600) as resp, open(deb_local, "wb") as out: + shutil.copyfileobj(resp, out) + except Exception as exc: + print(f"ERROR: could not download package binary: {exc}", file=sys.stderr) sys.exit(1) try: @@ -92,8 +113,9 @@ def run(package: str, version: str, arch: str = "arm64"): pr_body = ( f"## Remove package: `{package}` v{version}\n\n" f"Requested by @{user.login} (maintainer email: {maint_email}).\n\n" - f"File: `{file_path}`\n\n" - f"Submitted via `czdev unpublish`." + f"Manifest: `{file_path}`\n\n" + f"Submitted via `czdev unpublish`. Removing the manifest drops the package " + f"from the index on the next build; the apt-pool asset can be pruned separately." ) pr = gh.create_pull_request( TARGET_OWNER, TARGET_REPO, @@ -117,49 +139,3 @@ def extract_email(maintainer: str) -> str: if start != -1 and end != -1: return maintainer[start + 1:end] return maintainer - - -def fetch_deb_via_git(tmp_dir: Path, file_path: str): - """Sparse-checkout + LFS smudge a single deb file from the packages repo.""" - remote_url = f"git@github.com:{TARGET_OWNER}/{TARGET_REPO}.git" - cmds = [ - ["git", "init"], - ["git", "remote", "add", "origin", remote_url], - ["git", "lfs", "install", "--local"], - ["git", "config", "core.sparseCheckout", "true"], - ] - for cmd in cmds: - r = subprocess.run(cmd, cwd=str(tmp_dir), capture_output=True) - if r.returncode != 0: - return None - - # Write sparse-checkout pattern - sparse_dir = tmp_dir / ".git" / "info" - sparse_dir.mkdir(parents=True, exist_ok=True) - (sparse_dir / "sparse-checkout").write_text(file_path + "\n") - - # Fetch main (depth=1) then checkout - r = subprocess.run( - ["git", "fetch", "--depth=1", "origin", "main"], - cwd=str(tmp_dir), capture_output=True, - ) - if r.returncode != 0: - return None - - r = subprocess.run( - ["git", "checkout", "origin/main"], - cwd=str(tmp_dir), capture_output=True, - ) - if r.returncode != 0: - return None - - # LFS pull just this file - subprocess.run( - ["git", "lfs", "pull", "--include", file_path], - cwd=str(tmp_dir), capture_output=True, - ) - - local_path = tmp_dir / file_path - if local_path.exists() and local_path.stat().st_size > 200: - return local_path - return None From 26093e088bdd85a4163ab85207da00b3ec610b88 Mon Sep 17 00:00:00 2001 From: LiHaohua Date: Tue, 30 Jun 2026 19:26:51 +0800 Subject: [PATCH 2/2] ci: remove dead desktop-smoke workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit czdev was rewritten from Rust to Python and the crates/czdev Rust crate was removed, so 'cargo run -p czdev' (and the crates/czdev path filter) no longer resolve — this workflow has failed on every main push and PR since then. Remove it; the Python czdev has no Rust/emulator smoke harness. Co-authored-by: Cursor --- .github/workflows/desktop-smoke.yml | 104 ---------------------------- 1 file changed, 104 deletions(-) delete mode 100644 .github/workflows/desktop-smoke.yml diff --git a/.github/workflows/desktop-smoke.yml b/.github/workflows/desktop-smoke.yml deleted file mode 100644 index 767092c..0000000 --- a/.github/workflows/desktop-smoke.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Desktop smoke (czdev + emulator + hello_cz) - -on: - push: - branches: [main, dev, ci/**] - paths: - - 'crates/czdev/**' - - 'emulator' - - 'examples/hello_cz/**' - - 'sdk/**' - - '.github/workflows/desktop-smoke.yml' - pull_request: - branches: [main] - workflow_dispatch: - -jobs: - smoke: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, macos-14] - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install deps (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - build-essential cmake pkg-config \ - libsdl2-dev libsdl2-image-dev libsdl2-mixer-dev libfreetype-dev \ - xvfb - - - name: Install deps (macOS) - if: runner.os == 'macOS' - run: | - brew install cmake pkg-config sdl2 sdl2_image sdl2_mixer freetype - - - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: czdev doctor - run: cargo run -p czdev --release -- doctor - - - name: Build hello_cz (via czdev) - run: cargo run -p czdev --release -- build examples/hello_cz - - - name: Headless emulator smoke (Linux) - if: runner.os == 'Linux' - env: - SDL_VIDEODRIVER: dummy - SDL_AUDIODRIVER: dummy - run: | - set -e - cargo run -p czdev --release -- run examples/hello_cz & - EMU_PID=$! - sleep 5 - if ! kill -0 $EMU_PID 2>/dev/null; then - echo "::error::emulator exited before 5s"; exit 1 - fi - kill $EMU_PID || true - wait $EMU_PID 2>/dev/null || true - echo "emulator ran for 5s — OK" - - - name: Headless emulator smoke (macOS) - if: runner.os == 'macOS' - env: - SDL_VIDEODRIVER: dummy - SDL_AUDIODRIVER: dummy - run: | - set -e - cargo run -p czdev --release -- run examples/hello_cz & - EMU_PID=$! - sleep 5 - if ! kill -0 $EMU_PID 2>/dev/null; then - echo "::error::emulator exited before 5s"; exit 1 - fi - kill $EMU_PID || true - wait $EMU_PID 2>/dev/null || true - echo "emulator ran for 5s — OK" - - # Windows smoke is gated on the LVGL DLL rework (see DESKTOP_DEV.md §4 / T08). - # Keep an informational job so CI surfaces the status rather than failing silently. - windows-todo: - runs-on: windows-latest - steps: - - name: Windows desktop dev is pending - shell: bash - run: | - echo "Windows support for the czdev loop requires the emulator's" - echo "LVGL DLL rework (see docs/DESKTOP_DEV.md §4). Tracked as T08." - echo "This job is informational; it does not fail the workflow."