Skip to content

Commit e63b27f

Browse files
merynzmerynz
committed
ci: expand safe source rig scene import to disjoint 24 shards with eight concurrent hosted runners
1 parent 0e21b33 commit e63b27f

1 file changed

Lines changed: 200 additions & 0 deletions

File tree

‎.github/workflows/corpus4008-wave2-parallel-original-source-import-v1.yml‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
name: corpus4008-wave2-parallel-original-source-import-v1
2+
3+
# Import original SHA-verified quarantine files only. Output is observations,
4+
# NOT truth court, character admission, 2D art quality or model supervision.
5+
on:
6+
push:
7+
branches: [corpus4008-wave2-24shard-parallel-import-v1-20261107]
8+
paths:
9+
- '.github/workflows/corpus4008-wave2-parallel-original-source-import-v1.yml'
10+
- 'tools/corpus/plan_corpus4008_wave1_import_v1.py'
11+
- 'tools/corpus/blender_corpus4008_wave1_import_v1.py'
12+
- 'tests/corpus/test_corpus4008_wave1_source_import_v1.py'
13+
workflow_dispatch:
14+
15+
permissions:
16+
contents: read
17+
18+
concurrency:
19+
group: corpus4008-wave2-source-import-${{ github.ref }}
20+
cancel-in-progress: false
21+
22+
jobs:
23+
source-pilot-contract-tests:
24+
runs-on: ubuntu-24.04
25+
timeout-minutes: 10
26+
steps:
27+
- uses: actions/checkout@v4
28+
- name: Test deterministic sample, no-fake-PASS and holdout separation
29+
run: |
30+
python -m pip install --disable-pip-version-check pytest
31+
PYTHONPATH=. python -m pytest -q tests/corpus/test_corpus4008_wave1_source_import_v1.py
32+
python -m compileall -q \
33+
tools/corpus/plan_corpus4008_wave1_import_v1.py \
34+
tools/corpus/blender_corpus4008_wave1_import_v1.py
35+
bounded-original-source-import:
36+
needs: source-pilot-contract-tests
37+
if: github.event.repository.private == false
38+
runs-on: ubuntu-24.04
39+
timeout-minutes: 75
40+
strategy:
41+
fail-fast: false
42+
max-parallel: 8
43+
matrix:
44+
shard: [1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 17, 18, 19, 21, 22, 23, 25, 26, 27, 29, 30, 31]
45+
env:
46+
SHARD: ${{ matrix.shard }}
47+
RCLONE_CONFIG: /tmp/realsas-c4008-wave1-private.conf
48+
REMOTE: gdrive:RealSaS_MASTER_CORPUS_1024_V3
49+
PRIVATE_OUTPUT: gdrive:RealSaS_MASTER_CORPUS_1024_V3/audits/wave2_source_rig_scene_v1/${{ github.run_id }}-${{ github.run_attempt }}/shard-${{ matrix.shard }}
50+
PYTHONPATH: ${{ github.workspace }}
51+
steps:
52+
- uses: actions/checkout@v4
53+
- name: Require verified private Drive before any source inspection
54+
id: gate
55+
env:
56+
HAS_DRIVE: ${{ secrets.REALSAS_HOSTED_DRIVE_RCLONE_CONFIG_B64 != '' }}
57+
run: |
58+
if test "$HAS_DRIVE" = true; then
59+
echo 'ready=true' >> "$GITHUB_OUTPUT"
60+
else
61+
echo 'ready=false' >> "$GITHUB_OUTPUT"
62+
echo 'NO_PRIVATE_DRIVE__NO_IMPORTED_SOURCE_OR_TRUTH_CLAIMS' >> "$GITHUB_STEP_SUMMARY"
63+
fi
64+
- name: Acquire pinned original source receipts and Blender 4.5.14
65+
if: steps.gate.outputs.ready == 'true'
66+
env:
67+
DRIVE_CONFIG_B64: ${{ secrets.REALSAS_HOSTED_DRIVE_RCLONE_CONFIG_B64 }}
68+
run: |
69+
set -euo pipefail
70+
umask 077
71+
sudo apt-get update -qq
72+
sudo apt-get install -y -qq --no-install-recommends rclone xz-utils libegl1 libgl1 libgl1-mesa-dri libopengl0
73+
printf '%s' "$DRIVE_CONFIG_B64" | base64 -d > "$RCLONE_CONFIG"
74+
root="$RUNNER_TEMP/c4008-wave1"
75+
mkdir -p "$root/runtime" "$root/source" "$root/receipts" "$root/logs"
76+
rclone copyto "$REMOTE/audits/original_recovery_full_v1/42374046073-1/shard-$SHARD/SHARD_PRIVATE.json" \
77+
"$root/SHARD_PRIVATE.json" --retries 3
78+
python -m tools.corpus.plan_corpus4008_wave1_import_v1 \
79+
--shard-report "$root/SHARD_PRIVATE.json" \
80+
--shard-index "$SHARD" \
81+
--output "$root/PLAN.json" \
82+
--limit 8
83+
cd "$root/runtime"
84+
curl -fL --retry 4 --max-time 300 https://download.blender.org/release/Blender4.5/blender-4.5.14-linux-x64.tar.xz -o blender.tar.xz
85+
curl -fL --retry 4 --max-time 120 https://download.blender.org/release/Blender4.5/blender-4.5.14.sha256 -o official.sha256
86+
python - <<'PY'
87+
from pathlib import Path
88+
import hashlib,re
89+
lines=[x for x in Path('official.sha256').read_text().splitlines()
90+
if 'blender-4.5.14-linux-x64.tar.xz' in x]
91+
assert len(lines)==1
92+
expected=re.findall('[0-9a-fA-F]{64}',lines[0])
93+
assert len(expected)==1
94+
with open('blender.tar.xz','rb') as f:
95+
assert hashlib.file_digest(f,'sha256').hexdigest()==expected[0].lower()
96+
PY
97+
tar -xJf blender.tar.xz
98+
printf '050c02562f81fe80ba616a80198fa02d381e60f8b61b8d39add881f4bca0d7d8 %s\n' "$root/runtime/blender-4.5.14-linux-x64/blender" | sha256sum -c
99+
- name: Import selected originals in independent sandboxed Blender processes
100+
if: steps.gate.outputs.ready == 'true'
101+
run: |
102+
set -euo pipefail
103+
umask 077
104+
root="$RUNNER_TEMP/c4008-wave1"
105+
binary="$root/runtime/blender-4.5.14-linux-x64/blender"
106+
python - <<'PY'
107+
import json,os,re
108+
from pathlib import Path
109+
root=Path(os.environ['RUNNER_TEMP'])/'c4008-wave1'
110+
p=json.loads((root/'PLAN.json').read_text())
111+
with (root/'PLAN.tsv').open('w') as w:
112+
for x in p['entries']:
113+
sha=x['source_sha256'];ext=x['source_ext']
114+
aid=x['canonical_asset_id'];cid=x['candidate_id']
115+
assert re.fullmatch(r'[0-9a-f]{64}',sha)
116+
assert re.fullmatch(r'\.(fbx|glb|gltf)',ext)
117+
assert re.fullmatch(r'asset_[a-f0-9]{24}',aid)
118+
assert cid.endswith(sha)
119+
w.write('\t'.join((aid,cid,sha,ext,
120+
str(x['source_byte_count'])))+'\n')
121+
PY
122+
: > "$root/RESULTS.jsonl"
123+
while IFS="$(printf '\t')" read -r asset candidate source_sha ext source_bytes; do
124+
target="$root/source/$source_sha$ext"
125+
original="$REMOTE/audits/original_recovery_full_v1/42374046073-1/shard-$SHARD/verified_quarantine_objects/$source_sha$ext"
126+
status="SOURCE_FETCH_ERROR"
127+
if rclone copyto "$original" "$target" --retries 3 >"$root/logs/$asset.fetch.log" 2>&1; then
128+
status="SOURCE_SHA_OR_SIZE_MISMATCH"
129+
actual_sha="$(sha256sum "$target" | cut -d' ' -f1)"
130+
actual_size="$(stat -c %s "$target")"
131+
if test "$actual_sha" = "$source_sha" && test "$actual_size" = "$source_bytes"; then
132+
status="SOURCE_IMPORT_EXCEPTION"
133+
if timeout 150s "$binary" --background --factory-startup --disable-autoexec --python-exit-code 1 \
134+
--python tools/corpus/blender_corpus4008_wave1_import_v1.py -- \
135+
--source "$target" --source-sha256 "$source_sha" \
136+
--asset-id "$asset" --candidate-id "$candidate" \
137+
--output "$root/receipts/$asset.full.source.json.gz" \
138+
--summary "$root/receipts/$asset.summary.json" \
139+
>"$root/logs/$asset.blender.log" 2>&1; then
140+
status="IMPORTED_SOURCE_OBSERVATIONS_ONLY"
141+
fi
142+
fi
143+
fi
144+
export SRC_AID="$asset" SRC_SHA="$source_sha" SRC_STATUS="$status"
145+
python - <<'PY' >> "$root/RESULTS.jsonl"
146+
import json,os
147+
print(json.dumps({
148+
"asset_id":os.environ["SRC_AID"],
149+
"source_sha256":os.environ["SRC_SHA"],
150+
"status":os.environ["SRC_STATUS"],
151+
"truth_admitted":False,
152+
"training_authorized":False,
153+
},sort_keys=True))
154+
PY
155+
rm -f "$target"
156+
done < "$root/PLAN.tsv"
157+
python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
158+
import os,json
159+
from pathlib import Path
160+
from collections import Counter
161+
root=Path(os.environ["RUNNER_TEMP"])/'c4008-wave1'
162+
plan=json.loads((root/'PLAN.json').read_text())
163+
outcomes=[json.loads(x) for x in (root/'RESULTS.jsonl').read_text().splitlines()]
164+
assert len(outcomes)==plan['sample_size']
165+
assert len({x['asset_id'] for x in outcomes})==len(outcomes)
166+
assert all(x['training_authorized'] is False for x in outcomes)
167+
counts=Counter(x['status'] for x in outcomes)
168+
summary={
169+
"schema":"RealSaS.Corpus4008Wave1ImportBatchSummary.v1",
170+
"shard":int(os.environ['SHARD']),
171+
"selected_source_count":len(outcomes),
172+
"status_counts":dict(sorted(counts.items())),
173+
"truth_admitted":0,
174+
"training_authorized":False,
175+
}
176+
(root/'BATCH_STATUS.json').write_text(json.dumps(summary,indent=2)+"\n")
177+
print("### Original source importer wave — shard",os.environ['SHARD'])
178+
print("Selected:",len(outcomes),"Status:",dict(counts))
179+
print("Rig/mesh/skin/material labels are imported observations only.")
180+
PY
181+
- name: Store private immutable asset-level receipts and failure reasons
182+
if: always() && steps.gate.outputs.ready == 'true'
183+
run: |
184+
set -euo pipefail
185+
umask 077
186+
root="$RUNNER_TEMP/c4008-wave1"
187+
if test -s "$RCLONE_CONFIG" && test -f "$root/PLAN.json"; then
188+
rclone copyto "$root/PLAN.json" "$PRIVATE_OUTPUT/PLAN.json" --retries 3
189+
if test -f "$root/BATCH_STATUS.json"; then
190+
rclone copyto "$root/BATCH_STATUS.json" "$PRIVATE_OUTPUT/BATCH_STATUS.json" --retries 3
191+
fi
192+
if test -f "$root/RESULTS.jsonl"; then
193+
rclone copyto "$root/RESULTS.jsonl" "$PRIVATE_OUTPUT/RESULTS.jsonl" --retries 3
194+
fi
195+
rclone copy "$root/receipts" "$PRIVATE_OUTPUT/receipts" --checksum --retries 3
196+
# Blender import logs are held privately for diagnosis; never use
197+
# a log entry as model training admission or semantic teacher.
198+
rclone copy "$root/logs" "$PRIVATE_OUTPUT/logs" --checksum --retries 3
199+
echo 'Private imported-source observations written; zero head training admissions.' >> "$GITHUB_STEP_SUMMARY"
200+
fi

0 commit comments

Comments
 (0)