ci: release CI follow-ups: tighter coverage

.github/workflows/release-validation.yml

@@ -35,6 +35,9 @@ on:
       - 'v*.*.*-incubating-RC*'
   pull_request:
     types: [opened, synchronize, reopened]
+  schedule:
+    # Weekly run against main: catches dependency breakage between releases.
+    - cron: '0 9 * * 1'
   workflow_dispatch:
 
 concurrency:
@@ -113,8 +116,13 @@ jobs:
         if: steps.cache-rat.outputs.cache-hit != 'true'
         run: |
           mkdir -p ~/.cache/apache-rat
-          curl -fL -o ~/.cache/apache-rat/apache-rat-0.18.jar \
+          JAR="$HOME/.cache/apache-rat/apache-rat-0.18.jar"
+          curl -fL -o "$JAR" \
             https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.18/apache-rat-0.18.jar
+          # Verify integrity: SHA256 computed from the official Maven Central download
+          # and cross-checked against Maven Central's published SHA1.
+          echo "fe513ddd10cdc07e965ba430f2c093d8745ff24a0fb54efe0933653752c53301  $JAR" \
+            | sha256sum --check
 
       - name: Extract version
         id: version
@@ -194,13 +202,116 @@ jobs:
           retention-days: 7
           if-no-files-found: ignore
 
+  # Installs the wheel without any optional extras ([learn], etc.) and imports
+  # core symbols. Catches accidental leakage of optional dependencies into core
+  # code — a bare `pip install apache-burr` user would hit an ImportError that
+  # the [learn] smoke test would never see.
+  bare-install:
+    name: "Release Validation / bare-install"
+    needs: [check-paths, build-artifacts]
+    if: needs.check-paths.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Download release artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: release-artifacts
+          path: dist
+
+      - name: Install wheel without optional extras
+        env:
+          BURR_VERSION: ${{ needs.build-artifacts.outputs.version }}
+        run: |
+          pip install "dist/apache_burr-${BURR_VERSION}-py3-none-any.whl"
+
+      - name: Verify core imports succeed without optional dependencies
+        run: |
+          python -c "
+          import burr
+          from burr.core import ApplicationBuilder, State
+          from burr.core.action import action
+          print('Core imports OK')
+          "
+
+  # Extracts the sdist tarball, rebuilds the wheel from it (including the
+  # frontend npm build), then compares the resulting wheel's file contents
+  # against the release wheel using content hashes. Catches cases where the
+  # sdist is missing files that the direct wheel build includes.
+  sdist-wheel-equivalence:
+    name: "Release Validation / sdist-wheel-equivalence"
+    needs: [check-paths, build-artifacts]
+    if: needs.check-paths.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          cache: pip
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: npm
+          cache-dependency-path: telemetry/ui/package-lock.json
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Install system deps
+        run: sudo apt-get install -y --no-install-recommends graphviz
+
+      - name: Install Python build deps
+        run: pip install flit twine jinja2
+
+      - name: Download release artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: release-artifacts
+          path: dist
+
+      - name: Extract sdist and build wheel from it
+        env:
+          BURR_VERSION: ${{ needs.build-artifacts.outputs.version }}
+        run: |
+          mkdir -p /tmp/sdist-extract /tmp/sdist-wheel
+          tar -xzf "dist/apache-burr-${BURR_VERSION}-incubating-sdist.tar.gz" \
+            -C /tmp/sdist-extract
+          # Find the single top-level directory the tarball extracted into
+          SDIST_ROOT=$(find /tmp/sdist-extract -maxdepth 1 -mindepth 1 -type d | head -1)
+          cd "$SDIST_ROOT"
+          # Build wheel from within the extracted sdist. The sdist contains the
+          # React frontend source (telemetry/ui/) but not the compiled output,
+          # so the full npm build runs here — same as the original build.
+          python scripts/apache_release.py wheel "$BURR_VERSION" 0 \
+            --skip-signing --output-dir /tmp/sdist-wheel
+
+      - name: Compare sdist-built wheel against release wheel
+        env:
+          BURR_VERSION: ${{ needs.build-artifacts.outputs.version }}
+        run: |
+          python scripts/verify_apache_artifacts.py compare-wheels \
+            "dist/apache_burr-${BURR_VERSION}-py3-none-any.whl" \
+            "/tmp/sdist-wheel/apache_burr-${BURR_VERSION}-py3-none-any.whl"
+
   # Single stable required-check name. Always runs (if: always()) so it produces
   # a definite SUCCESS or FAILURE — never SKIPPED. Branch protection in
   # .asf.yaml requires this context, not the underlying jobs, so path-filtered
   # docs/website PRs (where the upstream jobs are skipped) still go green here.
   summary:
     name: "Release Validation / summary"
-    needs: [check-paths, build-artifacts, install-and-smoke]
+    needs: [check-paths, build-artifacts, install-and-smoke, bare-install, sdist-wheel-equivalence]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 2
@@ -210,13 +321,17 @@ jobs:
           CHECK_PATHS: ${{ needs.check-paths.result }}
           BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }}
           INSTALL_AND_SMOKE: ${{ needs.install-and-smoke.result }}
+          BARE_INSTALL: ${{ needs.bare-install.result }}
+          SDIST_WHEEL_EQUIV: ${{ needs.sdist-wheel-equivalence.result }}
         run: |
-          echo "check-paths:       $CHECK_PATHS"
-          echo "build-artifacts:   $BUILD_ARTIFACTS"
-          echo "install-and-smoke: $INSTALL_AND_SMOKE"
+          echo "check-paths:             $CHECK_PATHS"
+          echo "build-artifacts:         $BUILD_ARTIFACTS"
+          echo "install-and-smoke:       $INSTALL_AND_SMOKE"
+          echo "bare-install:            $BARE_INSTALL"
+          echo "sdist-wheel-equivalence: $SDIST_WHEEL_EQUIV"
           # Pass if every needed job is success or skipped; fail if any
           # failed or was cancelled.
-          for r in "$CHECK_PATHS" "$BUILD_ARTIFACTS" "$INSTALL_AND_SMOKE"; do
+          for r in "$CHECK_PATHS" "$BUILD_ARTIFACTS" "$INSTALL_AND_SMOKE" "$BARE_INSTALL" "$SDIST_WHEEL_EQUIV"; do
             case "$r" in
               success|skipped) ;;
               *) echo "::error::Release Validation failed (one or more jobs not success/skipped)"; exit 1 ;;

.pre-commit-config.yaml

@@ -78,3 +78,12 @@ repos:
         entry: npx --prefix telemetry/ui lint-staged
         pass_filenames: false
         always_run: true
+    -   id: check-asf-headers
+        name: Check ASF license headers
+        language: python
+        entry: python scripts/check_asf_headers.py
+        # Run on Python, YAML, and shell files — the source types that must
+        # carry the Apache 2.0 header. Exclusions are read from .rat-excludes
+        # at runtime so known third-party files are automatically respected.
+        types_or: [python, yaml, shell]
+        pass_filenames: true

.rat-excludes

@@ -44,15 +44,12 @@
 
 # Third-party MIT-licensed files (attributed in LICENSE).
 # Most names are unique within the repo so basename matching is safe.
-# Known collisions:
-#   - utils.py: also matches our own ASF code in burr/tracking/, etc.
-#     (4 other utils.py files; all currently have ASF headers)
+# Known collision:
 #   - button.tsx: also matches telemetry/ui/src/components/common/button.tsx
 #     (our own ASF code with header)
-# A future regression in any of those collision targets would silently pass
-# RAT. Tracked as a follow-up to rename or restructure.
+# A future regression in that collision target would silently pass RAT.
 **/prompts.py
-**/utils.py
+**/deep_researcher_utils.py
 **/animated-beam.tsx
 **/animated-shiny-text.tsx
 **/blur-fade.tsx

examples/deep-researcher/application.py

@@ -32,9 +32,9 @@
     import prompts
 
 try:
-    utils = importlib.import_module("burr.examples.deep-researcher.utils")
+    utils = importlib.import_module("burr.examples.deep-researcher.deep_researcher_utils")
 except ModuleNotFoundError:
-    import utils
+    import deep_researcher_utils as utils
 
 
 @functools.lru_cache

scripts/apache_release.py

@@ -1385,6 +1385,7 @@ def cmd_verify(args) -> bool:
     """Handle 'verify' subcommand."""
     _print_section(f"Verifying Artifacts - v{args.version}-RC{args.rc_num}")
 
+    skip_signing = getattr(args, "skip_signing", False)
     artifacts = _collect_all_artifacts(args.version, args.artifacts_dir)
 
     if not artifacts:
@@ -1395,7 +1396,7 @@ def cmd_verify(args) -> bool:
     for artifact in artifacts:
         if artifact.endswith((".asc", ".sha512")):
             continue  # Skip signature/checksum files
-        if not _verify_artifact_complete(artifact):
+        if not _verify_artifact_complete(artifact, skip_signing=skip_signing):
             all_valid = False
 
     if all_valid:
@@ -1594,6 +1595,11 @@ def _build_parser() -> argparse.ArgumentParser:
     verify_parser.add_argument("version", help="Version")
     verify_parser.add_argument("rc_num", help="RC number")
     verify_parser.add_argument("--artifacts-dir", default="dist")
+    verify_parser.add_argument(
+        "--skip-signing",
+        action="store_true",
+        help="Skip GPG signature verification (for builds produced with --skip-signing).",
+    )
 
     # vote-email subcommand
     vote_email_parser = subparsers.add_parser("vote-email", help="Generate release vote email")

scripts/check_asf_headers.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Check that Python, YAML, and shell files carry the ASF license header.
+
+Called by pre-commit with the list of staged files. Reads .rat-excludes at
+runtime so known third-party files are automatically respected without any
+duplication of the exclusion list.
+
+Usage (pre-commit invokes this automatically):
+    python scripts/check_asf_headers.py file1.py file2.yml ...
+"""
+
+import sys
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import Optional
+
+# Extensions whose source files must carry an ASF header.
+CHECKED_EXTENSIONS = {".py", ".yml", ".yaml", ".sh"}
+
+# Only search this many lines from the top of each file.
+# Headers are always at the start; searching the whole file would be slow
+# and would risk false positives from files that quote the license in prose.
+HEADER_SEARCH_LINES = 30
+
+# The one string that appears in every valid ASF license header regardless
+# of comment style (# for Python/YAML/shell, // for Java, /* for C, etc.).
+ASF_HEADER_MARKER = "Licensed to the Apache Software Foundation (ASF)"
+
+
+def _find_repo_root(start: Path) -> Path:
+    """Walk upward from start until we find .rat-excludes or pyproject.toml."""
+    for candidate in [start.resolve(), *start.resolve().parents]:
+        if (candidate / ".rat-excludes").exists() or (candidate / "pyproject.toml").exists():
+            return candidate
+    return start.resolve()
+
+
+def _load_rat_exclude_patterns(repo_root: Path) -> list:
+    """Return non-comment, non-blank lines from .rat-excludes as glob patterns."""
+    path = repo_root / ".rat-excludes"
+    if not path.exists():
+        return []
+    return [
+        line.strip()
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+
+
+def _is_excluded(file_path: Path, repo_root: Path, patterns: list) -> bool:
+    """Return True if file_path matches any pattern from .rat-excludes.
+
+    Patterns use RAT's **/<name> syntax. We handle this by checking the
+    file's basename against patterns that start with **/, and also checking
+    the full relative path against each pattern directly.
+    """
+    try:
+        rel = str(file_path.resolve().relative_to(repo_root.resolve()))
+    except ValueError:
+        rel = str(file_path)
+    name = file_path.name
+    for pattern in patterns:
+        if pattern.startswith("**/"):
+            # Strip the **/ prefix and match against the bare filename.
+            if fnmatch(name, pattern[3:]):
+                return True
+        if fnmatch(rel, pattern):
+            return True
+    return False
+
+
+def _has_asf_header(file_path: Path) -> bool:
+    """Return True if the ASF header marker appears within the first HEADER_SEARCH_LINES."""
+    try:
+        with file_path.open(encoding="utf-8", errors="replace") as fh:
+            for i, line in enumerate(fh):
+                if i >= HEADER_SEARCH_LINES:
+                    break
+                if ASF_HEADER_MARKER in line:
+                    return True
+    except OSError:
+        pass
+    return False
+
+
+def main(argv: Optional[list] = None) -> int:
+    files = [Path(p) for p in (argv if argv is not None else sys.argv[1:])]
+    if not files:
+        return 0
+
+    repo_root = _find_repo_root(files[0].parent)
+    patterns = _load_rat_exclude_patterns(repo_root)
+
+    violations = []
+    for f in files:
+        if f.suffix not in CHECKED_EXTENSIONS:
+            continue
+        if _is_excluded(f, repo_root, patterns):
+            continue
+        if not _has_asf_header(f):
+            violations.append(f)
+
+    if violations:
+        print("Missing ASF license header in the following file(s):")
+        for v in violations:
+            print(f"  {v}")
+        print()
+        print("Add the standard Apache 2.0 header block to each file.")
+        print("See any existing .py file in scripts/ for the correct format.")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())