|
| 1 | +# Copyright 2026 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Dynamic BCR Patches Downloader Engine. |
| 16 | +
|
| 17 | +This script resolves external dependencies dynamically on the target Windows VM. |
| 18 | +It parses the JSON Bzlmod dependency graph computed on-the-fly by `bazel mod graph`, |
| 19 | +recursively queries the Bazel Central Registry (BCR), extracts registry metadata |
| 20 | +and source files, downloads all transitive registry patches (which must be applied |
| 21 | +locally), isolates colliding patch names into separate directories, and outputs |
| 22 | +the dynamically computed `--distdir` flags to be loaded directly by the Windows batch script. |
| 23 | +""" |
| 24 | + |
| 25 | +import json |
| 26 | +import os |
| 27 | +import sys |
| 28 | +import time |
| 29 | +import urllib.error |
| 30 | +import urllib.request |
| 31 | + |
| 32 | +def find_modules(node, modules=None): |
| 33 | + """Recursively parses Bzlmod JSON module graph to extract all module names/versions.""" |
| 34 | + if modules is None: |
| 35 | + modules = set() |
| 36 | + |
| 37 | + if isinstance(node, dict): |
| 38 | + name = node.get("name") |
| 39 | + version = node.get("version") |
| 40 | + if name and version and name != "cel-python" and name != "cel-cpp": |
| 41 | + modules.add((name, version)) |
| 42 | + |
| 43 | + for val in node.values(): |
| 44 | + find_modules(val, modules) |
| 45 | + elif isinstance(node, list): |
| 46 | + for item in node: |
| 47 | + find_modules(item, modules) |
| 48 | + |
| 49 | + return modules |
| 50 | + |
| 51 | +def urlopen_with_retry( |
| 52 | + url, headers=None, timeout=30, max_retries=5, backoff_factor=2 |
| 53 | +): |
| 54 | + """Executes urlopen with exponential backoff retry logic to absorb VM drops.""" |
| 55 | + if headers is None: |
| 56 | + headers = {"User-Agent": "Mozilla/5.0"} |
| 57 | + |
| 58 | + req = urllib.request.Request(url, headers=headers) |
| 59 | + |
| 60 | + retries = 0 |
| 61 | + delay = 1 |
| 62 | + while True: |
| 63 | + try: |
| 64 | + # Use standard urlopen (respects proxy settings) |
| 65 | + return urllib.request.urlopen(req, timeout=timeout) |
| 66 | + except (urllib.error.URLError, ConnectionError, TimeoutError, Exception) as e: |
| 67 | + retries += 1 |
| 68 | + if retries > max_retries: |
| 69 | + print(f"Failed to fetch {url} after {max_retries} attempts.", file=sys.stderr) |
| 70 | + raise e |
| 71 | + print(f"Error fetching {url} (attempt {retries}/{max_retries}): {e}. Retrying in {delay}s...", file=sys.stderr) |
| 72 | + time.sleep(delay) |
| 73 | + delay *= backoff_factor |
| 74 | + |
| 75 | +def main(): |
| 76 | + if len(sys.argv) < 3: |
| 77 | + print("Usage: python download_patches.py <graph.json> <distdir_base_path>", file=sys.stderr) |
| 78 | + sys.exit(1) |
| 79 | + |
| 80 | + graph_path = sys.argv[1] |
| 81 | + distdir_base = sys.argv[2] |
| 82 | + |
| 83 | + if not os.path.exists(graph_path): |
| 84 | + print(f"Error: {graph_path} not found.", file=sys.stderr) |
| 85 | + sys.exit(1) |
| 86 | + |
| 87 | + with open(graph_path, "r") as f: |
| 88 | + graph = json.load(f) |
| 89 | + |
| 90 | + modules = find_modules(graph) |
| 91 | + print(f"Found {len(modules)} unique transitive modules.", file=sys.stderr) |
| 92 | + |
| 93 | + patches_found = [] |
| 94 | + |
| 95 | + for name, version in sorted(modules): |
| 96 | + url = f"https://bcr.bazel.build/modules/{name}/{version}/source.json" |
| 97 | + try: |
| 98 | + with urlopen_with_retry(url, timeout=30) as response: |
| 99 | + source_data = json.loads(response.read().decode()) |
| 100 | + patches = source_data.get("patches") |
| 101 | + if patches: |
| 102 | + print(f"Module {name}@{version} has patches: {list(patches.keys())}", file=sys.stderr) |
| 103 | + for patch_name in patches.keys(): |
| 104 | + patch_url = f"https://bcr.bazel.build/modules/{name}/{version}/patches/{patch_name}" |
| 105 | + patches_found.append({ |
| 106 | + "module": name, |
| 107 | + "version": version, |
| 108 | + "patch_name": patch_name, |
| 109 | + "url": patch_url |
| 110 | + }) |
| 111 | + except urllib.error.HTTPError as e: |
| 112 | + if e.code != 404: |
| 113 | + print(f"HTTP Error for {name}@{version}: {e.code}", file=sys.stderr) |
| 114 | + except Exception as e: |
| 115 | + print(f"Error fetching {name}@{version}: {e}", file=sys.stderr) |
| 116 | + |
| 117 | + print(f"Found {len(patches_found)} patches to download.", file=sys.stderr) |
| 118 | + |
| 119 | + distdir_dirs = set() |
| 120 | + |
| 121 | + for patch in patches_found: |
| 122 | + module_clean = patch["module"].replace("-", "_") |
| 123 | + dir_name = os.path.join(distdir_base, module_clean) |
| 124 | + distdir_dirs.add(dir_name) |
| 125 | + |
| 126 | + os.makedirs(dir_name, exist_ok=True) |
| 127 | + dest_file = os.path.join(dir_name, patch["patch_name"]) |
| 128 | + |
| 129 | + print(f"Downloading {patch['url']} to {dest_file}...", file=sys.stderr) |
| 130 | + try: |
| 131 | + with ( |
| 132 | + urlopen_with_retry(patch["url"], timeout=30) as response, |
| 133 | + open(dest_file, "wb") as out, |
| 134 | + ): |
| 135 | + out.write(response.read()) |
| 136 | + except Exception as e: |
| 137 | + print(f"Failed to download {patch['url']}: {e}", file=sys.stderr) |
| 138 | + sys.exit(1) |
| 139 | + |
| 140 | + # Generate the env setting command for batch |
| 141 | + distdir_flags = [] |
| 142 | + for d in sorted(distdir_dirs): |
| 143 | + d_win = d.replace("/", "\\") |
| 144 | + distdir_flags.append(f"--distdir={d_win}") |
| 145 | + |
| 146 | + flags_str = " ".join(distdir_flags) |
| 147 | + print(f'@set "DISTDIR_FLAGS={flags_str}"') |
| 148 | + |
| 149 | +if __name__ == "__main__": |
| 150 | + main() |
0 commit comments