|
| 1 | +name: corpus4008-wave1-source-rig-scene-import-v1 |
| 2 | + |
| 3 | +# Import original SHA-verified quarantine files only. Output is observations, |
| 4 | +# NOT truth court, character admission, 2D art quality or model supervision. |
| 5 | +on: |
| 6 | + push: |
| 7 | + branches: [corpus4008-wave1-source-rig-semantic-inventory-v1-20261107] |
| 8 | + paths: |
| 9 | + - '.github/workflows/corpus4008-wave1-source-rig-scene-import-v1.yml' |
| 10 | + - 'tools/corpus/plan_corpus4008_wave1_import_v1.py' |
| 11 | + - 'tools/corpus/blender_corpus4008_wave1_import_v1.py' |
| 12 | + - 'tests/corpus/test_corpus4008_wave1_source_import_v1.py' |
| 13 | + workflow_dispatch: |
| 14 | + |
| 15 | +permissions: |
| 16 | + contents: read |
| 17 | + |
| 18 | +concurrency: |
| 19 | + group: corpus4008-wave1-source-import-${{ github.ref }} |
| 20 | + cancel-in-progress: true |
| 21 | + |
| 22 | +jobs: |
| 23 | + source-pilot-contract-tests: |
| 24 | + runs-on: ubuntu-24.04 |
| 25 | + timeout-minutes: 10 |
| 26 | + steps: |
| 27 | + - uses: actions/checkout@v4 |
| 28 | + - name: Test deterministic sample, no-fake-PASS and holdout separation |
| 29 | + run: | |
| 30 | + python -m pip install --disable-pip-version-check pytest |
| 31 | + PYTHONPATH=. python -m pytest -q tests/corpus/test_corpus4008_wave1_source_import_v1.py |
| 32 | + python -m compileall -q \ |
| 33 | + tools/corpus/plan_corpus4008_wave1_import_v1.py \ |
| 34 | + tools/corpus/blender_corpus4008_wave1_import_v1.py |
| 35 | + bounded-original-source-import: |
| 36 | + needs: source-pilot-contract-tests |
| 37 | + if: github.event.repository.private == false |
| 38 | + runs-on: ubuntu-24.04 |
| 39 | + timeout-minutes: 75 |
| 40 | + strategy: |
| 41 | + fail-fast: false |
| 42 | + max-parallel: 4 |
| 43 | + matrix: |
| 44 | + shard: [0, 4, 8, 12, 16, 20, 24, 28] |
| 45 | + env: |
| 46 | + SHARD: ${{ matrix.shard }} |
| 47 | + RCLONE_CONFIG: /tmp/realsas-c4008-wave1-private.conf |
| 48 | + REMOTE: gdrive:RealSaS_MASTER_CORPUS_1024_V3 |
| 49 | + PRIVATE_OUTPUT: gdrive:RealSaS_MASTER_CORPUS_1024_V3/audits/wave1_source_rig_scene_v1/${{ github.run_id }}-${{ github.run_attempt }}/shard-${{ matrix.shard }} |
| 50 | + PYTHONPATH: ${{ github.workspace }} |
| 51 | + steps: |
| 52 | + - uses: actions/checkout@v4 |
| 53 | + - name: Require verified private Drive before any source inspection |
| 54 | + id: gate |
| 55 | + env: |
| 56 | + HAS_DRIVE: ${{ secrets.REALSAS_HOSTED_DRIVE_RCLONE_CONFIG_B64 != '' }} |
| 57 | + run: | |
| 58 | + if test "$HAS_DRIVE" = true; then |
| 59 | + echo 'ready=true' >> "$GITHUB_OUTPUT" |
| 60 | + else |
| 61 | + echo 'ready=false' >> "$GITHUB_OUTPUT" |
| 62 | + echo 'NO_PRIVATE_DRIVE__NO_IMPORTED_SOURCE_OR_TRUTH_CLAIMS' >> "$GITHUB_STEP_SUMMARY" |
| 63 | + fi |
| 64 | + - name: Acquire pinned original source receipts and Blender 4.5.14 |
| 65 | + if: steps.gate.outputs.ready == 'true' |
| 66 | + env: |
| 67 | + DRIVE_CONFIG_B64: ${{ secrets.REALSAS_HOSTED_DRIVE_RCLONE_CONFIG_B64 }} |
| 68 | + run: | |
| 69 | + set -euo pipefail |
| 70 | + umask 077 |
| 71 | + sudo apt-get update -qq |
| 72 | + sudo apt-get install -y -qq --no-install-recommends rclone xz-utils libegl1 libgl1 libgl1-mesa-dri libopengl0 |
| 73 | + printf '%s' "$DRIVE_CONFIG_B64" | base64 -d > "$RCLONE_CONFIG" |
| 74 | + root="$RUNNER_TEMP/c4008-wave1" |
| 75 | + mkdir -p "$root/runtime" "$root/source" "$root/receipts" "$root/logs" |
| 76 | + rclone copyto "$REMOTE/audits/original_recovery_full_v1/42374046073-1/shard-$SHARD/SHARD_PRIVATE.json" \ |
| 77 | + "$root/SHARD_PRIVATE.json" --retries 3 |
| 78 | + python -m tools.corpus.plan_corpus4008_wave1_import_v1 \ |
| 79 | + --shard-report "$root/SHARD_PRIVATE.json" \ |
| 80 | + --shard-index "$SHARD" \ |
| 81 | + --output "$root/PLAN.json" \ |
| 82 | + --limit 8 |
| 83 | + cd "$root/runtime" |
| 84 | + curl -fL --retry 4 --max-time 300 https://download.blender.org/release/Blender4.5/blender-4.5.14-linux-x64.tar.xz -o blender.tar.xz |
| 85 | + curl -fL --retry 4 --max-time 120 https://download.blender.org/release/Blender4.5/blender-4.5.14.sha256 -o official.sha256 |
| 86 | + python - <<'PY' |
| 87 | + from pathlib import Path |
| 88 | + import hashlib,re |
| 89 | + lines=[x for x in Path('official.sha256').read_text().splitlines() |
| 90 | + if 'blender-4.5.14-linux-x64.tar.xz' in x] |
| 91 | + assert len(lines)==1 |
| 92 | + expected=re.findall('[0-9a-fA-F]{64}',lines[0]) |
| 93 | + assert len(expected)==1 |
| 94 | + with open('blender.tar.xz','rb') as f: |
| 95 | + assert hashlib.file_digest(f,'sha256').hexdigest()==expected[0].lower() |
| 96 | + PY |
| 97 | + tar -xJf blender.tar.xz |
| 98 | + printf '050c02562f81fe80ba616a80198fa02d381e60f8b61b8d39add881f4bca0d7d8 %s\n' "$root/runtime/blender-4.5.14-linux-x64/blender" | sha256sum -c |
| 99 | + - name: Import selected originals in independent sandboxed Blender processes |
| 100 | + if: steps.gate.outputs.ready == 'true' |
| 101 | + run: | |
| 102 | + set -euo pipefail |
| 103 | + umask 077 |
| 104 | + root="$RUNNER_TEMP/c4008-wave1" |
| 105 | + binary="$root/runtime/blender-4.5.14-linux-x64/blender" |
| 106 | + python - <<'PY' |
| 107 | + import json,os,re |
| 108 | + from pathlib import Path |
| 109 | + root=Path(os.environ['RUNNER_TEMP'])/'c4008-wave1' |
| 110 | + p=json.loads((root/'PLAN.json').read_text()) |
| 111 | + with (root/'PLAN.tsv').open('w') as w: |
| 112 | + for x in p['entries']: |
| 113 | + sha=x['source_sha256'];ext=x['source_ext'] |
| 114 | + aid=x['canonical_asset_id'];cid=x['candidate_id'] |
| 115 | + assert re.fullmatch(r'[0-9a-f]{64}',sha) |
| 116 | + assert re.fullmatch(r'\.(fbx|glb|gltf)',ext) |
| 117 | + assert re.fullmatch(r'asset_[a-f0-9]{24}',aid) |
| 118 | + assert cid.endswith(sha) |
| 119 | + w.write('\t'.join((aid,cid,sha,ext, |
| 120 | + str(x['source_byte_count'])))+'\n') |
| 121 | + PY |
| 122 | + : > "$root/RESULTS.jsonl" |
| 123 | + while IFS="$(printf '\t')" read -r asset candidate source_sha ext source_bytes; do |
| 124 | + target="$root/source/$source_sha$ext" |
| 125 | + original="$REMOTE/audits/original_recovery_full_v1/42374046073-1/shard-$SHARD/verified_quarantine_objects/$source_sha$ext" |
| 126 | + status="SOURCE_FETCH_ERROR" |
| 127 | + if rclone copyto "$original" "$target" --retries 3 >"$root/logs/$asset.fetch.log" 2>&1; then |
| 128 | + status="SOURCE_SHA_OR_SIZE_MISMATCH" |
| 129 | + actual_sha="$(sha256sum "$target" | cut -d' ' -f1)" |
| 130 | + actual_size="$(stat -c %s "$target")" |
| 131 | + if test "$actual_sha" = "$source_sha" && test "$actual_size" = "$source_bytes"; then |
| 132 | + status="SOURCE_IMPORT_EXCEPTION" |
| 133 | + if timeout 150s "$binary" --background --factory-startup --disable-autoexec --python-exit-code 1 \ |
| 134 | + --python tools/corpus/blender_corpus4008_wave1_import_v1.py -- \ |
| 135 | + --source "$target" --source-sha256 "$source_sha" \ |
| 136 | + --asset-id "$asset" --candidate-id "$candidate" \ |
| 137 | + --output "$root/receipts/$asset.full.source.json.gz" \ |
| 138 | + --summary "$root/receipts/$asset.summary.json" \ |
| 139 | + >"$root/logs/$asset.blender.log" 2>&1; then |
| 140 | + status="IMPORTED_SOURCE_OBSERVATIONS_ONLY" |
| 141 | + fi |
| 142 | + fi |
| 143 | + fi |
| 144 | + export SRC_AID="$asset" SRC_SHA="$source_sha" SRC_STATUS="$status" |
| 145 | + python - <<'PY' >> "$root/RESULTS.jsonl" |
| 146 | + import json,os |
| 147 | + print(json.dumps({ |
| 148 | + "asset_id":os.environ["SRC_AID"], |
| 149 | + "source_sha256":os.environ["SRC_SHA"], |
| 150 | + "status":os.environ["SRC_STATUS"], |
| 151 | + "truth_admitted":False, |
| 152 | + "training_authorized":False, |
| 153 | + },sort_keys=True)) |
| 154 | + PY |
| 155 | + rm -f "$target" |
| 156 | + done < "$root/PLAN.tsv" |
| 157 | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" |
| 158 | + import os,json |
| 159 | + from pathlib import Path |
| 160 | + from collections import Counter |
| 161 | + root=Path(os.environ["RUNNER_TEMP"])/'c4008-wave1' |
| 162 | + plan=json.loads((root/'PLAN.json').read_text()) |
| 163 | + outcomes=[json.loads(x) for x in (root/'RESULTS.jsonl').read_text().splitlines()] |
| 164 | + assert len(outcomes)==plan['sample_size'] |
| 165 | + assert len({x['asset_id'] for x in outcomes})==len(outcomes) |
| 166 | + assert all(x['training_authorized'] is False for x in outcomes) |
| 167 | + counts=Counter(x['status'] for x in outcomes) |
| 168 | + summary={ |
| 169 | + "schema":"RealSaS.Corpus4008Wave1ImportBatchSummary.v1", |
| 170 | + "shard":int(os.environ['SHARD']), |
| 171 | + "selected_source_count":len(outcomes), |
| 172 | + "status_counts":dict(sorted(counts.items())), |
| 173 | + "truth_admitted":0, |
| 174 | + "training_authorized":False, |
| 175 | + } |
| 176 | + (root/'BATCH_STATUS.json').write_text(json.dumps(summary,indent=2)+"\n") |
| 177 | + print("### Original source importer wave — shard",os.environ['SHARD']) |
| 178 | + print("Selected:",len(outcomes),"Status:",dict(counts)) |
| 179 | + print("Rig/mesh/skin/material labels are imported observations only.") |
| 180 | + PY |
| 181 | + - name: Store private immutable asset-level receipts and failure reasons |
| 182 | + if: always() && steps.gate.outputs.ready == 'true' |
| 183 | + run: | |
| 184 | + set -euo pipefail |
| 185 | + umask 077 |
| 186 | + root="$RUNNER_TEMP/c4008-wave1" |
| 187 | + if test -s "$RCLONE_CONFIG" && test -f "$root/PLAN.json"; then |
| 188 | + rclone copyto "$root/PLAN.json" "$PRIVATE_OUTPUT/PLAN.json" --retries 3 |
| 189 | + if test -f "$root/BATCH_STATUS.json"; then |
| 190 | + rclone copyto "$root/BATCH_STATUS.json" "$PRIVATE_OUTPUT/BATCH_STATUS.json" --retries 3 |
| 191 | + fi |
| 192 | + if test -f "$root/RESULTS.jsonl"; then |
| 193 | + rclone copyto "$root/RESULTS.jsonl" "$PRIVATE_OUTPUT/RESULTS.jsonl" --retries 3 |
| 194 | + fi |
| 195 | + rclone copy "$root/receipts" "$PRIVATE_OUTPUT/receipts" --checksum --retries 3 |
| 196 | + # Blender import logs are held privately for diagnosis; never use |
| 197 | + # a log entry as model training admission or semantic teacher. |
| 198 | + rclone copy "$root/logs" "$PRIVATE_OUTPUT/logs" --checksum --retries 3 |
| 199 | + echo 'Private imported-source observations written; zero head training admissions.' >> "$GITHUB_STEP_SUMMARY" |
| 200 | + fi |
0 commit comments