From 2b8f8259ef4b52bdd25ebca822e108edc63c71a8 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:38:25 +0000 Subject: [PATCH 01/14] feat(prism): emit ExecutionProof in work-unit result payload (M3 prism-proof) Add src/prism_challenge/proof.py: ExecutionProof v1 envelope (schema-compatible with the base worker plane), canonical manifest hashing (sha256 over json.dumps(manifest, sort_keys=True, indent=2) == the exact on-disk bytes of prism_run_manifest.v2.json), tier computation from injected provider env (tier 0 default; tier 1 needs BOTH image digest AND pod metadata; tier 2 only with a non-null attestation payload), and an sr25519 worker signature over the pinned message sha256(f'{manifest_sha256}:{unit_id}') (identical to the base agent, VAL-AGENT-008/VAL-PRISM-006). Wire emission into the validator finalization path: the dispatch result payload carries exactly one signed proof at successful finalization, gated on a new prism worker_plane config block (enabled by default OFF -> legacy behavior). Proof construction reads only the manifest + a fixed non-secret provider env allowlist and never the held-out split config or LLM keys (VAL-PRISM-008). Fulfills VAL-PRISM-001/002/003/004/005/006/008. --- src/prism_challenge/config.py | 30 +- src/prism_challenge/proof.py | 385 ++++++++++++++++++++++ src/prism_challenge/repository.py | 19 ++ src/prism_challenge/validator_dispatch.py | 10 +- src/prism_challenge/validator_executor.py | 95 +++++- tests/test_prism_proof.py | 327 ++++++++++++++++++ tests/test_prism_proof_emission.py | 245 ++++++++++++++ 7 files changed, 1104 insertions(+), 7 deletions(-) create mode 100644 src/prism_challenge/proof.py create mode 100644 tests/test_prism_proof.py create mode 100644 tests/test_prism_proof_emission.py diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index 32ade58..922d6b4 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -3,17 +3,43 @@ from pathlib import Path from typing import Literal -from pydantic import AliasChoices, Field +from pydantic import AliasChoices, BaseModel, Field from pydantic_settings import SettingsConfigDict from .sdk.config import ChallengeSettings +class WorkerPlaneConfig(BaseModel): + """Prism worker-plane feature block (architecture.md 3.4/3.5). + + OFF by default: with ``enabled`` false prism behaves exactly as before the compute plane (no + ExecutionProof emission, no admission gate, legacy audit-free finalization). Env overrides use + the nested delimiter, e.g. ``PRISM_WORKER_PLANE__ENABLED=true``. + """ + + enabled: bool = False + admission_requires_worker: bool = False + audit_rate_tier0: float = Field(default=0.10, ge=0.0, le=1.0) + audit_rate_tier1: float = Field(default=0.05, ge=0.0, le=1.0) + audit_rate_tier2: float = Field(default=0.02, ge=0.0, le=1.0) + # sr25519 signing key (URI ``//Name`` / mnemonic / seed) for the worker that emits + # ExecutionProofs. This is the worker's OWN key, injected by the worker agent -- NEVER a + # master-side secret. Unset -> prism emits no signed proof (the base worker plane may still + # stamp a tier-0 proof from the manifest hash). + signing_key: str | None = Field(default=None, repr=False) + + class PrismSettings(ChallengeSettings): model_config = SettingsConfigDict( - env_prefix="PRISM_", env_file=".env", extra="ignore", populate_by_name=True + env_prefix="PRISM_", + env_file=".env", + extra="ignore", + populate_by_name=True, + env_nested_delimiter="__", ) + worker_plane: WorkerPlaneConfig = Field(default_factory=WorkerPlaneConfig) + database_url: str = Field( default="sqlite+aiosqlite:////data/prism.sqlite3", validation_alias=AliasChoices("PRISM_DATABASE_URL", "CHALLENGE_DATABASE_URL"), diff --git a/src/prism_challenge/proof.py b/src/prism_challenge/proof.py new file mode 100644 index 0000000..d6df9d3 --- /dev/null +++ b/src/prism_challenge/proof.py @@ -0,0 +1,385 @@ +"""ExecutionProof construction and verification for the prism evaluator (architecture.md 3.4). + +The prism evaluator emits an :class:`ExecutionProof` in the work-unit result payload at every +successful finalization. The tier-0 core is the deterministic ``manifest_sha256`` -- the sha256 of +the canonical on-disk bytes of ``prism_run_manifest.v2.json`` (``json.dumps(manifest, +sort_keys=True, indent=2)``, the exact form the runner/host persist) -- plus the worker's sr25519 +signature binding that hash to the work unit. The signed message format is PINNED identically to the +base worker plane (VAL-AGENT-008 / VAL-PRISM-006): the signature is over +``sha256(manifest_sha256 + ":" + unit_id)`` -- the sha256 digest of the UTF-8 bytes of the string +``{manifest_sha256}:{unit_id}`` -- so a proof prism emits verifies with the same code as one the +base worker plane emits, and a proof cannot be replayed across units. + +Tiers (architecture.md 3.4): + +* tier 0 -- mandatory, all backends: canonical manifest hash + worker signature. +* tier 1 -- BOTH a pinned ``image_digest`` AND pod metadata (``provider.pod_id``). +* tier 2 -- a non-null ``attestation`` payload carrying ``tdx_quote_b64`` and/or ``gpu_eat_jwt``. + +Security invariant (VAL-PRISM-008): proof construction reads ONLY the manifest, the work unit id, +the worker signer, and a FIXED ALLOWLIST of non-secret provider env vars. It never reads the +master-side held-out split configuration (``base_eval_val_data_dir`` / heldout config) or any LLM +provider key -- there is no code path from proof construction to those secrets (this module imports +no settings and touches no filesystem path other than an explicitly supplied manifest file). +""" + +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel + +from .auth import verify_hotkey_signature + +EXECUTION_PROOF_VERSION = 1 + +#: Result-payload key carrying the serialized :class:`ExecutionProof`. Identical to the base worker +#: plane key so a proof prism emits is passed through unchanged by the base ``WorkerProofExecutor``. +PROOF_PAYLOAD_KEY = "execution_proof" + +# Non-secret provider env vars the worker agent injects for the tier-1/2 fields (architecture 3.5). +PROVIDER_NAME_ENV = "PRISM_PROVIDER_NAME" +EXECUTOR_ID_ENV = "PRISM_EXECUTOR_ID" +POD_ID_ENV = "PRISM_POD_ID" +MINER_HOTKEY_ENV = "PRISM_MINER_HOTKEY" +IMAGE_DIGEST_ENV = "PRISM_IMAGE_DIGEST" +ATTESTATION_ENV = "PRISM_ATTESTATION" + +#: The ONLY env vars proof construction ever reads (all non-secret provider provenance). +PROVIDER_ENV_KEYS: tuple[str, ...] = ( + PROVIDER_NAME_ENV, + EXECUTOR_ID_ENV, + POD_ID_ENV, + MINER_HOTKEY_ENV, + IMAGE_DIGEST_ENV, + ATTESTATION_ENV, +) + +#: Documented attestation payload keys (architecture.md 3.4). +ATTESTATION_KEYS: tuple[str, ...] = ("tdx_quote_b64", "gpu_eat_jwt") + + +class ProviderInfo(BaseModel): + """Provider/pod identity carried by an ExecutionProof (architecture 3.4).""" + + name: str + executor_id: str | None = None + pod_id: str | None = None + miner_hotkey: str | None = None + + +class WorkerSignature(BaseModel): + """The worker's sr25519 signature binding a manifest hash to a work unit.""" + + worker_pubkey: str + sig: str + + +class ExecutionProof(BaseModel): + """Proof envelope attached to every worker result (architecture 3.4). + + The schema is byte-compatible with the base worker plane's ``ExecutionProof`` so a proof prism + emits round-trips through the platform result path unchanged. + """ + + version: int = EXECUTION_PROOF_VERSION + tier: int = 0 + manifest_sha256: str + image_digest: str | None = None + provider: ProviderInfo | None = None + worker_signature: WorkerSignature + attestation: dict[str, Any] | None = None + + +@runtime_checkable +class WorkerSigner(Protocol): + """Signs the pinned ExecutionProof message with a worker sr25519 key.""" + + @property + def worker_pubkey(self) -> str: ... + + def sign(self, message: bytes) -> str: ... + + +@dataclass(frozen=True) +class KeypairWorkerSigner: + """A :class:`WorkerSigner` backed by a substrate (bittensor) keypair.""" + + keypair: Any + + @property + def worker_pubkey(self) -> str: + return str(self.keypair.ss58_address) + + def sign(self, message: bytes) -> str: + signature = self.keypair.sign(message) + if isinstance(signature, bytes | bytearray): + return "0x" + bytes(signature).hex() + return str(signature) + + +def worker_signer_from_key(key: str) -> KeypairWorkerSigner: + """Build a worker signer from an sr25519 URI (``//Name``), mnemonic, or seed. + + The key is the WORKER's OWN signing key (never a master secret); the worker agent injects it. + """ + + import bittensor as bt + + value = key.strip() + if value.startswith("//"): + keypair = bt.Keypair.create_from_uri(value) + elif " " in value: + keypair = bt.Keypair.create_from_mnemonic(value) + else: + keypair = bt.Keypair.create_from_seed(value) + return KeypairWorkerSigner(keypair) + + +def canonical_manifest_json(manifest: Mapping[str, Any]) -> str: + """The canonical on-disk serialization of a run manifest (``sort_keys=True, indent=2``). + + Key-order-insensitive by construction: two dicts with identical content but different key order + serialize identically, so the manifest hash is stable regardless of how the manifest is built. + """ + + return json.dumps(manifest, sort_keys=True, indent=2) + + +def compute_manifest_sha256(manifest: Mapping[str, Any]) -> str: + """sha256 hex of the canonical manifest bytes (order-insensitive).""" + + return manifest_sha256_from_bytes(canonical_manifest_json(manifest).encode("utf-8")) + + +def manifest_sha256_from_bytes(raw: bytes) -> str: + """sha256 hex of raw manifest bytes (use the exact on-disk bytes of the v2 manifest).""" + + return hashlib.sha256(raw).hexdigest() + + +def read_manifest_sha256(path: str | os.PathLike[str]) -> str: + """sha256 hex of the exact on-disk bytes of the manifest file at ``path``.""" + + return manifest_sha256_from_bytes(Path(path).read_bytes()) + + +def execution_proof_signing_payload(*, manifest_sha256: str, unit_id: str) -> bytes: + """The exact bytes an ExecutionProof signature covers (pinned format). + + ``sha256`` digest of the UTF-8 bytes of ``{manifest_sha256}:{unit_id}``. + """ + + return hashlib.sha256(f"{manifest_sha256}:{unit_id}".encode()).digest() + + +def has_attestation(attestation: Any) -> bool: + """Whether ``attestation`` is a populated payload of the documented shape (tier-2 gate).""" + + return isinstance(attestation, Mapping) and any( + attestation.get(key) for key in ATTESTATION_KEYS + ) + + +def compute_tier( + *, + image_digest: str | None, + provider: ProviderInfo | None, + attestation: Any, +) -> int: + """Compute the proof tier from the available provenance (architecture 3.4). + + tier 2 iff a populated attestation payload is present; else tier 1 iff BOTH a pinned image + digest AND pod metadata (``provider.pod_id``) are present; else tier 0. + """ + + if has_attestation(attestation): + return 2 + if image_digest and provider is not None and provider.pod_id: + return 1 + return 0 + + +def provider_from_env(env: Mapping[str, str] | None = None) -> ProviderInfo | None: + """Read the provider block from the injected provider env (``None`` when no provider name).""" + + env = os.environ if env is None else env + name = _clean(env.get(PROVIDER_NAME_ENV)) + if not name: + return None + return ProviderInfo( + name=name, + executor_id=_clean(env.get(EXECUTOR_ID_ENV)), + pod_id=_clean(env.get(POD_ID_ENV)), + miner_hotkey=_clean(env.get(MINER_HOTKEY_ENV)), + ) + + +def image_digest_from_env(env: Mapping[str, str] | None = None) -> str | None: + """Read the evaluator image digest from the injected provider env.""" + + env = os.environ if env is None else env + return _clean(env.get(IMAGE_DIGEST_ENV)) + + +def attestation_from_env(env: Mapping[str, str] | None = None) -> dict[str, Any] | None: + """Read + parse the attestation payload (JSON) from the injected provider env, if any.""" + + env = os.environ if env is None else env + raw = _clean(env.get(ATTESTATION_ENV)) + if not raw: + return None + try: + parsed = json.loads(raw) + except (ValueError, TypeError): + return None + return parsed if isinstance(parsed, dict) else None + + +def build_execution_proof( + *, + signer: WorkerSigner, + manifest_sha256: str, + unit_id: str, + provider: ProviderInfo | None = None, + image_digest: str | None = None, + attestation: dict[str, Any] | None = None, + tier: int | None = None, +) -> ExecutionProof: + """Build and sign an ExecutionProof binding ``manifest_sha256`` to ``unit_id`` under ``signer``. + + The tier is computed from the provenance unless explicitly overridden. ``signer`` is the WORKER + keypair; its public identity becomes ``worker_signature.worker_pubkey``. + """ + + effective_tier = ( + compute_tier(image_digest=image_digest, provider=provider, attestation=attestation) + if tier is None + else tier + ) + signature = signer.sign( + execution_proof_signing_payload(manifest_sha256=manifest_sha256, unit_id=unit_id) + ) + return ExecutionProof( + version=EXECUTION_PROOF_VERSION, + tier=effective_tier, + manifest_sha256=manifest_sha256, + image_digest=image_digest, + provider=provider, + worker_signature=WorkerSignature(worker_pubkey=signer.worker_pubkey, sig=signature), + attestation=attestation, + ) + + +def build_execution_proof_from_manifest( + *, + signer: WorkerSigner, + unit_id: str, + manifest: Mapping[str, Any] | None = None, + manifest_bytes: bytes | None = None, + manifest_path: str | os.PathLike[str] | None = None, + env: Mapping[str, str] | None = None, +) -> ExecutionProof: + """Build a signed proof from a manifest source + the injected provider env. + + Exactly ONE manifest source must be given. Prefer ``manifest_path`` at emission time so the + hash is taken from the exact on-disk bytes of ``prism_run_manifest.v2.json``. The provider + provenance is read ONLY from the non-secret provider env allowlist. + """ + + digest = _resolve_manifest_sha256( + manifest=manifest, manifest_bytes=manifest_bytes, manifest_path=manifest_path + ) + return build_execution_proof( + signer=signer, + manifest_sha256=digest, + unit_id=unit_id, + provider=provider_from_env(env), + image_digest=image_digest_from_env(env), + attestation=attestation_from_env(env), + ) + + +def verify_execution_proof( + proof: ExecutionProof, + *, + unit_id: str, + verify: Any = verify_hotkey_signature, +) -> bool: + """Whether ``proof``'s worker signature verifies for ``unit_id`` (sr25519, pinned message). + + Rejects a proof presented with a DIFFERENT ``unit_id`` than the one signed, so a proof cannot + be replayed across units. + """ + + payload = execution_proof_signing_payload( + manifest_sha256=proof.manifest_sha256, unit_id=unit_id + ) + return bool( + verify(proof.worker_signature.worker_pubkey, payload, proof.worker_signature.sig) + ) + + +def _resolve_manifest_sha256( + *, + manifest: Mapping[str, Any] | None, + manifest_bytes: bytes | None, + manifest_path: str | os.PathLike[str] | None, +) -> str: + provided = [item for item in (manifest, manifest_bytes, manifest_path) if item is not None] + if len(provided) != 1: + raise ValueError( + "exactly one of manifest, manifest_bytes, manifest_path is required" + ) + if manifest is not None: + return compute_manifest_sha256(manifest) + if manifest_bytes is not None: + return manifest_sha256_from_bytes(manifest_bytes) + assert manifest_path is not None + return read_manifest_sha256(manifest_path) + + +def _clean(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +__all__ = [ + "ATTESTATION_ENV", + "ATTESTATION_KEYS", + "EXECUTION_PROOF_VERSION", + "EXECUTOR_ID_ENV", + "IMAGE_DIGEST_ENV", + "MINER_HOTKEY_ENV", + "POD_ID_ENV", + "PROOF_PAYLOAD_KEY", + "PROVIDER_ENV_KEYS", + "PROVIDER_NAME_ENV", + "ExecutionProof", + "KeypairWorkerSigner", + "ProviderInfo", + "WorkerSignature", + "WorkerSigner", + "attestation_from_env", + "build_execution_proof", + "build_execution_proof_from_manifest", + "canonical_manifest_json", + "compute_manifest_sha256", + "compute_tier", + "execution_proof_signing_payload", + "has_attestation", + "image_digest_from_env", + "manifest_sha256_from_bytes", + "provider_from_env", + "read_manifest_sha256", + "verify_execution_proof", + "worker_signer_from_key", +] diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index 99d6c1d..e68f6c8 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -1042,6 +1042,25 @@ async def container_job_attempt_count(self, submission_id: str, level: str) -> i ) return int(list(rows)[0]["count"]) + async def latest_run_manifest_path(self, submission_id: str, level: str) -> str | None: + """On-disk path of the most recent eval job's ``prism_run_manifest.v2.json`` for a run. + + Used at successful finalization to hash the exact on-disk manifest bytes for the + ExecutionProof (architecture.md 3.4). Returns ``None`` when no eval job recorded a manifest. + """ + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT run_manifest_path FROM eval_jobs WHERE submission_id=? AND level=? " + "AND run_manifest_path IS NOT NULL " + "ORDER BY attempts DESC, created_at DESC LIMIT 1", + (submission_id, level), + ) + row_list = list(rows) + if not row_list: + return None + value = row_list[0]["run_manifest_path"] + return str(value) if value is not None else None + async def latest_retryable_container_job( self, submission_id: str, level: str ) -> dict[str, object] | None: diff --git a/src/prism_challenge/validator_dispatch.py b/src/prism_challenge/validator_dispatch.py index a4f315e..15a6b19 100644 --- a/src/prism_challenge/validator_dispatch.py +++ b/src/prism_challenge/validator_dispatch.py @@ -31,6 +31,7 @@ from .config import PrismSettings from .config import settings as default_settings from .evaluator.checkpoint_publisher import CheckpointPublisher +from .proof import PROOF_PAYLOAD_KEY from .validator_executor import run_validator_cycle CHALLENGE_SLUG = "prism" @@ -80,12 +81,19 @@ async def dispatch_assignment( summary = await run_validator_cycle(worker=app.state.worker, work_unit_ids=[work_unit_id]) finally: await database.close() - return { + result: dict[str, Any] = { "pulled": summary.pulled, "executed": summary.executed, "skipped": summary.skipped, "completed_submissions": list(summary.completed_submissions), } + # Emit the ExecutionProof IN the work-unit result payload at successful finalization + # (architecture.md 3.4; VAL-PRISM-001). Absent when the worker plane is off or the unit did not + # freshly finalize. + proof = summary.execution_proofs.get(work_unit_id) + if proof is not None: + result[PROOF_PAYLOAD_KEY] = proof + return result def gateway_scoped_settings( diff --git a/src/prism_challenge/validator_executor.py b/src/prism_challenge/validator_executor.py index 539a366..c4f6ffb 100644 --- a/src/prism_challenge/validator_executor.py +++ b/src/prism_challenge/validator_executor.py @@ -14,8 +14,11 @@ from __future__ import annotations -from collections.abc import Iterable -from dataclasses import dataclass +import logging +import os +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any from .coordination import ( PRISM_DEFAULT_CONCURRENCY, @@ -25,8 +28,15 @@ list_pending_prism_work_units, pull_assigned_work_units, ) +from .proof import ( + WorkerSigner, + build_execution_proof_from_manifest, + worker_signer_from_key, +) from .queue import PrismWorker +logger = logging.getLogger(__name__) + #: Submission statuses at which a prism work unit is terminal (no re-execution, no re-dispatch). TERMINAL_SUBMISSION_STATUSES = frozenset({"completed", "failed", "rejected"}) @@ -42,6 +52,9 @@ class PrismWorkUnitExecution: executed: bool #: True when a fresh result was persisted by this run (False = already terminal). posted: bool + #: The serialized ExecutionProof emitted for this unit's result payload, when the worker plane + #: is enabled and a fresh successful finalization produced a manifest to bind (else ``None``). + execution_proof: dict[str, Any] | None = None @dataclass(frozen=True) @@ -52,15 +65,26 @@ class PrismValidatorCycleSummary: executed: int skipped: int completed_submissions: tuple[str, ...] + #: ExecutionProofs emitted this cycle, keyed by ``work_unit_id`` (empty when the plane is off). + execution_proofs: dict[str, dict[str, Any]] = field(default_factory=dict) -async def execute_work_unit(worker: PrismWorker, unit: PrismWorkUnit) -> PrismWorkUnitExecution: +async def execute_work_unit( + worker: PrismWorker, + unit: PrismWorkUnit, + *, + proof_signer: WorkerSigner | None = None, + proof_env: Mapping[str, str] | None = None, +) -> PrismWorkUnitExecution: """Run one assigned prism re-execution on the validator's own broker and report the outcome. Idempotent: :meth:`PrismWorker.process_submission` claims the submission only while it is pending, so a unit that already reached a terminal state is not re-dispatched and its recorded result is left untouched. A reassigned unit carrying ``resume_checkpoint_ref`` in its payload resumes from the last public HF checkpoint instead of restarting (VAL-PRISM-023). + + When the worker plane is enabled, a fresh successful finalization also emits an + :class:`~prism_challenge.proof.ExecutionProof` bound to this unit (architecture.md 3.4). """ resume_ref = unit.payload.get(RESUME_CHECKPOINT_PAYLOAD_KEY) @@ -70,15 +94,70 @@ async def execute_work_unit(worker: PrismWorker, unit: PrismWorkUnit) -> PrismWo ) executed = result_id is not None status = await worker.repository.submission_status(unit.submission_id) + execution_proof: dict[str, Any] | None = None + if executed and status == "completed": + execution_proof = await _emit_execution_proof( + worker, unit, signer=proof_signer, env=proof_env + ) return PrismWorkUnitExecution( work_unit_id=unit.work_unit_id, submission_id=unit.submission_id, status=status or "", executed=executed, posted=executed, + execution_proof=execution_proof, ) +async def _emit_execution_proof( + worker: PrismWorker, + unit: PrismWorkUnit, + *, + signer: WorkerSigner | None, + env: Mapping[str, str] | None, +) -> dict[str, Any] | None: + """Build the ExecutionProof for a freshly finalized unit, or ``None`` when not applicable. + + Gated on ``worker_plane.enabled``. The manifest hash is taken from the exact on-disk bytes of + the run's ``prism_run_manifest.v2.json``; the provenance comes ONLY from the non-secret provider + env allowlist. Held-out split config and LLM keys are never read (VAL-PRISM-008). + """ + + if not worker.settings.worker_plane.enabled: + return None + resolved_signer = _resolve_proof_signer(worker, signer) + if resolved_signer is None: + logger.warning( + "worker plane enabled but no signing key configured; no ExecutionProof emitted " + "for work_unit=%s", + unit.work_unit_id, + ) + return None + manifest_path = await worker.repository.latest_run_manifest_path( + unit.submission_id, worker.execution_backend + ) + if not manifest_path: + return None + proof = build_execution_proof_from_manifest( + signer=resolved_signer, + unit_id=unit.work_unit_id, + manifest_path=manifest_path, + env=os.environ if env is None else env, + ) + return proof.model_dump(mode="json") + + +def _resolve_proof_signer( + worker: PrismWorker, signer: WorkerSigner | None +) -> WorkerSigner | None: + if signer is not None: + return signer + key = worker.settings.worker_plane.signing_key + if not key: + return None + return worker_signer_from_key(key) + + async def run_validator_cycle( *, worker: PrismWorker, @@ -86,6 +165,8 @@ async def run_validator_cycle( capabilities: Iterable[str] = (PRISM_WORK_UNIT_CAPABILITY,), in_flight: int | None = None, max_concurrency: int = PRISM_DEFAULT_CONCURRENCY, + proof_signer: WorkerSigner | None = None, + proof_env: Mapping[str, str] | None = None, ) -> PrismValidatorCycleSummary: """Run one decentralized validator cycle: pull -> execute (own broker) -> post. @@ -112,17 +193,23 @@ async def run_validator_cycle( executed = 0 skipped = 0 completed: list[str] = [] + execution_proofs: dict[str, dict[str, Any]] = {} for unit in pulled: - outcome = await execute_work_unit(worker, unit) + outcome = await execute_work_unit( + worker, unit, proof_signer=proof_signer, proof_env=proof_env + ) if outcome.executed: executed += 1 else: skipped += 1 if outcome.status == "completed": completed.append(outcome.submission_id) + if outcome.execution_proof is not None: + execution_proofs[outcome.work_unit_id] = outcome.execution_proof return PrismValidatorCycleSummary( pulled=len(pulled), executed=executed, skipped=skipped, completed_submissions=tuple(completed), + execution_proofs=execution_proofs, ) diff --git a/tests/test_prism_proof.py b/tests/test_prism_proof.py new file mode 100644 index 0000000..711ca46 --- /dev/null +++ b/tests/test_prism_proof.py @@ -0,0 +1,327 @@ +"""ExecutionProof construction/verification unit tests (VAL-PRISM-002/003/004/005/006/008). + +Offline, no GPU: exercises ``prism_challenge.proof`` directly with REAL sr25519 keypairs +(bittensor) so the pinned signature format is verified end-to-end, and pins the canonical manifest +hashing, tier computation, and the held-out-secret security invariant. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge import proof +from prism_challenge.auth import verify_hotkey_signature +from prism_challenge.proof import ( + ExecutionProof, + build_execution_proof, + build_execution_proof_from_manifest, + compute_manifest_sha256, + verify_execution_proof, + worker_signer_from_key, +) + +UNIT_ID = "submission-abc123" + +MANIFEST_A: dict[str, Any] = { + "schema_version": "prism_run_manifest.v2", + "run": {"world_size": 1, "nproc_per_node": 1, "device": "cpu"}, + "metrics": {"prequential_bpb": 1.2345, "available_bytes": 4096.0}, + "score": {"final_score": 0.42}, +} +MANIFEST_B: dict[str, Any] = { + "schema_version": "prism_run_manifest.v2", + "metrics": {"prequential_bpb": 0.9}, + "compute": {"gpu_count": 1, "world_size": 1, "nproc_per_node": 1, "device": "cpu"}, + "artifacts": {"trained_state": "trained_state.pt"}, +} + + +def _signer(uri: str = "//WorkerAlice") -> proof.KeypairWorkerSigner: + return worker_signer_from_key(uri) + + +# --- VAL-PRISM-002: canonical manifest hashing (both directions + order-insensitive) ------------ + + +@pytest.mark.parametrize("manifest", [MANIFEST_A, MANIFEST_B]) +def test_manifest_sha256_matches_on_disk_bytes_both_ways( + manifest: dict[str, Any], tmp_path: Path +) -> None: + signer = _signer() + # Persist the manifest exactly as the runner/host do (canonical: sort_keys=True, indent=2). + path = tmp_path / "prism_run_manifest.v2.json" + path.write_text(json.dumps(manifest, sort_keys=True, indent=2), encoding="utf-8") + + built = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest_path=path + ) + + # (i) sha256 of the EXACT on-disk bytes, computed OUTSIDE the proof code path. + disk_digest = hashlib.sha256(path.read_bytes()).hexdigest() + # (ii) sha256 of json.dumps(json.loads(file), sort_keys=True, indent=2) bytes. + reloaded = json.loads(path.read_text(encoding="utf-8")) + canonical_digest = hashlib.sha256( + json.dumps(reloaded, sort_keys=True, indent=2).encode("utf-8") + ).hexdigest() + + assert built.manifest_sha256 == disk_digest + assert built.manifest_sha256 == canonical_digest + assert len(built.manifest_sha256) == 64 + assert built.manifest_sha256 == built.manifest_sha256.lower() + assert all(c in "0123456789abcdef" for c in built.manifest_sha256) + + +def test_manifest_hash_is_key_order_insensitive() -> None: + reordered = {key: MANIFEST_A[key] for key in reversed(list(MANIFEST_A))} + assert list(reordered) != list(MANIFEST_A) + assert compute_manifest_sha256(reordered) == compute_manifest_sha256(MANIFEST_A) + signer = _signer() + p1 = build_execution_proof_from_manifest(signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A) + p2 = build_execution_proof_from_manifest(signer=signer, unit_id=UNIT_ID, manifest=reordered) + assert p1.manifest_sha256 == p2.manifest_sha256 + + +def test_manifest_source_is_exactly_one() -> None: + signer = _signer() + with pytest.raises(ValueError): + build_execution_proof_from_manifest(signer=signer, unit_id=UNIT_ID) + with pytest.raises(ValueError): + build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, manifest_bytes=b"{}" + ) + + +# --- VAL-PRISM-003: tier 0 with no provider metadata ------------------------------------------- + + +def test_tier0_when_no_provider_metadata() -> None: + signer = _signer() + digest = compute_manifest_sha256(MANIFEST_A) + empty_env: dict[str, str] = {} + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=empty_env + ) + assert p.tier == 0 + assert p.image_digest is None + assert p.provider is None + assert p.attestation is None + assert p.manifest_sha256 == digest + + +# --- VAL-PRISM-004: tier 1 requires BOTH image digest AND pod metadata -------------------------- + +_IMAGE_DIGEST = "sha256:" + "c" * 64 +_FULL_ENV = { + proof.PROVIDER_NAME_ENV: "lium", + proof.EXECUTOR_ID_ENV: "ex-1", + proof.POD_ID_ENV: "pod-1", + proof.MINER_HOTKEY_ENV: "miner-H1", + proof.IMAGE_DIGEST_ENV: _IMAGE_DIGEST, +} + + +def test_tier1_full_provider_env() -> None: + signer = _signer() + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=dict(_FULL_ENV) + ) + assert p.tier == 1 + assert p.image_digest == _IMAGE_DIGEST + assert p.provider is not None + assert p.provider.name == "lium" + assert p.provider.executor_id == "ex-1" + assert p.provider.pod_id == "pod-1" + assert p.provider.miner_hotkey == "miner-H1" + + +def test_tier1_downgrades_to_tier0_without_pod_or_digest() -> None: + signer = _signer() + digest_only = {k: v for k, v in _FULL_ENV.items() if k != proof.POD_ID_ENV} + pod_only = {k: v for k, v in _FULL_ENV.items() if k != proof.IMAGE_DIGEST_ENV} + p_digest_only = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=digest_only + ) + p_pod_only = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=pod_only + ) + assert p_digest_only.tier == 0 + assert p_pod_only.tier == 0 + + +# --- VAL-PRISM-005: tier 2 only with a non-null attestation payload ----------------------------- + + +def test_tier2_only_with_attestation_payload() -> None: + signer = _signer() + attestation = {"tdx_quote_b64": "QUOTE", "gpu_eat_jwt": "JWT"} + env = {**_FULL_ENV, proof.ATTESTATION_ENV: json.dumps(attestation)} + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=env + ) + assert p.tier == 2 + # payload preserved verbatim + assert p.attestation == attestation + + +def test_tier_never_2_without_attestation() -> None: + signer = _signer() + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=dict(_FULL_ENV) + ) + assert p.attestation is None + assert p.tier != 2 + + +def test_attestation_object_missing_documented_keys_is_not_tier2() -> None: + signer = _signer() + env = {**_FULL_ENV, proof.ATTESTATION_ENV: json.dumps({"unrelated": "x"})} + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=env + ) + assert p.tier != 2 + assert p.tier == 1 # falls back to the digest+pod tier + + +# --- VAL-PRISM-006: worker signature binds manifest_sha256 + unit_id ---------------------------- + + +def test_worker_signature_verifies_over_pinned_message() -> None: + signer = _signer() + p = build_execution_proof( + signer=signer, manifest_sha256=compute_manifest_sha256(MANIFEST_A), unit_id=UNIT_ID + ) + assert verify_execution_proof(p, unit_id=UNIT_ID) is True + # Independent verifier given only (proof JSON, unit_id) accepts the signature. + reloaded = ExecutionProof.model_validate_json(p.model_dump_json()) + assert verify_execution_proof(reloaded, unit_id=UNIT_ID) is True + # Independent raw sr25519 check over the pinned message. + payload = proof.execution_proof_signing_payload( + manifest_sha256=p.manifest_sha256, unit_id=UNIT_ID + ) + assert verify_hotkey_signature( + p.worker_signature.worker_pubkey, payload, p.worker_signature.sig + ) + + +def test_signing_payload_is_sha256_of_pinned_string() -> None: + digest = compute_manifest_sha256(MANIFEST_A) + payload = proof.execution_proof_signing_payload(manifest_sha256=digest, unit_id=UNIT_ID) + assert payload == hashlib.sha256(f"{digest}:{UNIT_ID}".encode()).digest() + + +def test_proof_cannot_be_replayed_across_units() -> None: + signer = _signer() + p = build_execution_proof( + signer=signer, manifest_sha256=compute_manifest_sha256(MANIFEST_A), unit_id=UNIT_ID + ) + assert verify_execution_proof(p, unit_id="a-different-unit") is False + + +def test_tampered_hash_or_signature_fails_verification() -> None: + signer = _signer() + p = build_execution_proof( + signer=signer, manifest_sha256=compute_manifest_sha256(MANIFEST_A), unit_id=UNIT_ID + ) + tampered_hash = p.model_copy(update={"manifest_sha256": "b" * 64}) + assert verify_execution_proof(tampered_hash, unit_id=UNIT_ID) is False + tampered_sig = p.model_copy( + update={ + "worker_signature": p.worker_signature.model_copy(update={"sig": "0x" + "00" * 64}) + } + ) + assert verify_execution_proof(tampered_sig, unit_id=UNIT_ID) is False + + +# --- VAL-PRISM-008: proof construction cannot read held-out secret split config ----------------- + +# Env keys that would leak master-side secrets if the builder ever read them. +_SECRET_ENV_KEYS = frozenset( + { + "PRISM_BASE_EVAL_VAL_DATA_DIR", + "PRISM_EVAL_VAL_DATA_DIR", + "PRISM_GATEWAY_TOKEN", + "BASE_GATEWAY_TOKEN", + "PRISM_HF_TOKEN", + } +) +_SECRET_VAL_PATH = "/data/fineweb-edu/val/SECRET-HELDOUT-SPLIT" +_SECRET_LLM_KEY = "sk-SECRET-LLM-PROVIDER-KEY" + + +class _SecretAccessError(RuntimeError): + """Raised by the sentinel env when a held-out-secret key is accessed.""" + + +class _SentinelEnv(dict): + """A mapping that RAISES on any access to a held-out-secret key. + + Proof construction that only reads the non-secret provider allowlist never trips it; a builder + that reached for the val-split path / LLM key would raise, proving the read path exists. + """ + + def __getitem__(self, key: str) -> Any: + if key in _SECRET_ENV_KEYS: + raise _SecretAccessError(key) + return super().__getitem__(key) + + def get(self, key: str, default: Any = None) -> Any: # type: ignore[override] + if key in _SECRET_ENV_KEYS: + raise _SecretAccessError(key) + return super().get(key, default) + + +def test_proof_construction_succeeds_with_heldout_config_unset() -> None: + # (a) No provider/secret env at all: proof still builds (tier 0). + signer = _signer() + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env={} + ) + assert p.version == 1 + assert p.tier == 0 + + +def test_proof_construction_never_touches_secret_env_keys() -> None: + # (b) A sentinel that raises on secret-key access is armed, yet proof construction succeeds. + signer = _signer() + sentinel = _SentinelEnv( + { + **_FULL_ENV, + "PRISM_BASE_EVAL_VAL_DATA_DIR": _SECRET_VAL_PATH, + "PRISM_GATEWAY_TOKEN": _SECRET_LLM_KEY, + "PRISM_HF_TOKEN": _SECRET_LLM_KEY, + } + ) + # The sentinel is genuinely armed: reaching for a secret key raises. + with pytest.raises(_SecretAccessError): + sentinel.get("PRISM_BASE_EVAL_VAL_DATA_DIR") + + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=sentinel + ) + assert p.tier == 1 # only the non-secret provider allowlist was consulted + + +def test_serialized_proof_leaks_no_secret_material() -> None: + # (c) The serialized proof carries no val-split path or LLM key material. + signer = _signer() + sentinel = _SentinelEnv( + { + **_FULL_ENV, + "PRISM_BASE_EVAL_VAL_DATA_DIR": _SECRET_VAL_PATH, + "PRISM_GATEWAY_TOKEN": _SECRET_LLM_KEY, + } + ) + p = build_execution_proof_from_manifest( + signer=signer, unit_id=UNIT_ID, manifest=MANIFEST_A, env=sentinel + ) + serialized = p.model_dump_json() + assert _SECRET_VAL_PATH not in serialized + assert "SECRET-HELDOUT-SPLIT" not in serialized + assert _SECRET_LLM_KEY not in serialized + # The proof module only ever reads the non-secret provider allowlist. + assert not (set(proof.PROVIDER_ENV_KEYS) & _SECRET_ENV_KEYS) diff --git a/tests/test_prism_proof_emission.py b/tests/test_prism_proof_emission.py new file mode 100644 index 0000000..dffbb2a --- /dev/null +++ b/tests/test_prism_proof_emission.py @@ -0,0 +1,245 @@ +"""ExecutionProof emission in the work-unit result payload (VAL-PRISM-001, and 002 end-to-end). + +Offline, no GPU: drives a real prism finalization via the CPU re-exec mock (the same seam +``test_validator_dispatch`` uses) and asserts that a successful finalization emits exactly one +version-1 ExecutionProof IN the dispatch result payload, that its ``manifest_sha256`` equals the +independently computed hash of the on-disk ``prism_run_manifest.v2.json``, and that the worker +signature verifies. Also pins the flag-OFF regression (no proof, legacy payload). +""" + +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import zipfile +from pathlib import Path + +from prism_challenge.app import create_app +from prism_challenge.auth import verify_hotkey_signature +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + PROOF_PAYLOAD_KEY, + ExecutionProof, + execution_proof_signing_payload, + verify_execution_proof, +) +from prism_challenge.validator_dispatch import dispatch_assignment + +BROKER_URL = "http://broker-val:8082" +WORKER_KEY = "//WorkerEmitter" + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=worker_plane, + ) + + +def _payload() -> dict[str, str]: + return { + "gateway_url": "http://master:8081", + "BASE_LLM_GATEWAY_URL": "http://master:8081/llm/v1", + "gateway_token": "scoped-token", + } + + +async def _seed(settings: PrismSettings, hotkey: str) -> str: + app = create_app(settings) + await app.state.database.init() + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + await app.state.database.close() + return sub.id + + +def _on_disk_manifest(settings: PrismSettings) -> Path: + matches = list(Path(settings.base_eval_artifact_root).rglob("prism_run_manifest.v2.json")) + assert len(matches) == 1, f"expected exactly one manifest, found {matches}" + return matches[0] + + +async def test_finalization_emits_version1_proof_in_result_payload(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + # Provider env the worker agent would inject (tier-1 provenance). + monkeypatch.setenv("PRISM_PROVIDER_NAME", "lium") + monkeypatch.setenv("PRISM_EXECUTOR_ID", "ex-77") + monkeypatch.setenv("PRISM_POD_ID", "pod-77") + monkeypatch.setenv("PRISM_MINER_HOTKEY", "miner-owner") + monkeypatch.setenv("PRISM_IMAGE_DIGEST", "sha256:" + "d" * 64) + + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + submission_id = await _seed(settings, "hk-owner") + + result = await dispatch_assignment( + work_unit_id=submission_id, + payload=_payload(), + broker_url=BROKER_URL, + settings=settings, + ) + + assert result["executed"] == 1 + # Exactly one proof, IN the result payload. + assert PROOF_PAYLOAD_KEY in result + proof_json = result[PROOF_PAYLOAD_KEY] + proof = ExecutionProof.model_validate(proof_json) + assert proof.version == 1 + assert isinstance(proof.tier, int) + assert len(proof.manifest_sha256) == 64 + assert proof.manifest_sha256 == proof.manifest_sha256.lower() + assert proof.worker_signature.worker_pubkey + assert proof.worker_signature.sig + + # VAL-PRISM-002 end-to-end: manifest_sha256 equals the independent hash of the on-disk bytes, + # computed both ways. + manifest_path = _on_disk_manifest(settings) + disk_digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + reloaded = json.loads(manifest_path.read_text(encoding="utf-8")) + canonical_digest = hashlib.sha256( + json.dumps(reloaded, sort_keys=True, indent=2).encode("utf-8") + ).hexdigest() + assert proof.manifest_sha256 == disk_digest == canonical_digest + + # Tier 1 provenance echoed from the injected env. + assert proof.tier == 1 + assert proof.image_digest == "sha256:" + "d" * 64 + assert proof.provider is not None + assert proof.provider.name == "lium" + assert proof.provider.pod_id == "pod-77" + assert proof.provider.miner_hotkey == "miner-owner" + + # Signature verifies against the pinned message for THIS unit and not another. + assert verify_execution_proof(proof, unit_id=submission_id) is True + assert verify_execution_proof(proof, unit_id="some-other-unit") is False + payload = execution_proof_signing_payload( + manifest_sha256=proof.manifest_sha256, unit_id=submission_id + ) + assert verify_hotkey_signature( + proof.worker_signature.worker_pubkey, payload, proof.worker_signature.sig + ) + + +async def test_no_proof_emitted_when_worker_plane_disabled(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings(tmp_path, worker_plane=WorkerPlaneConfig(enabled=False)) + submission_id = await _seed(settings, "hk-owner") + + result = await dispatch_assignment( + work_unit_id=submission_id, + payload=_payload(), + broker_url=BROKER_URL, + settings=settings, + ) + + assert result["executed"] == 1 + assert PROOF_PAYLOAD_KEY not in result + + +async def test_no_proof_when_enabled_without_signing_key(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=None) + ) + submission_id = await _seed(settings, "hk-owner") + + result = await dispatch_assignment( + work_unit_id=submission_id, + payload=_payload(), + broker_url=BROKER_URL, + settings=settings, + ) + + assert result["executed"] == 1 + assert PROOF_PAYLOAD_KEY not in result From 3aa005b02de0c5c0f1cefd79eacb456508a572f4 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:11:47 +0000 Subject: [PATCH 02/14] feat(prism): add base->prism result accept route with proof verification + idempotent ingestion (M3 prism-proof) CREATE POST /internal/v1/work_units/result (body {work_unit_id, submission_ref, result}), the counterpart to base's HttpChallengeResultForwarder. Verify the forwarded ExecutionProof BEFORE scoring: reject missing/malformed proof (VAL-PRISM-018) and tampered manifest / forged digest (VAL-PRISM-007) 422 with a distinguishable reason, leaving the unit eligible for retry. Downgrade unverifiable tier claims to their effective tier for audit sampling (VAL-PRISM-019). Finalize idempotently via a guarded work_unit_results key so a duplicate delivery is a no-op and a conflicting one is refused 409, never mutating a finalized score/leaderboard (VAL-PRISM-017). Gated behind worker_plane.enabled (404 when off). --- src/prism_challenge/app.py | 59 +++ src/prism_challenge/audit.py | 148 ++++++ src/prism_challenge/config.py | 5 + src/prism_challenge/db.py | 16 + src/prism_challenge/ingestion.py | 299 ++++++++++++ src/prism_challenge/proof.py | 6 + src/prism_challenge/repository.py | 51 ++ src/prism_challenge/validator_dispatch.py | 8 +- src/prism_challenge/validator_executor.py | 40 +- tests/test_prism_audit_effective_tier.py | 139 ++++++ tests/test_prism_result_ingestion.py | 546 ++++++++++++++++++++++ 11 files changed, 1307 insertions(+), 10 deletions(-) create mode 100644 src/prism_challenge/audit.py create mode 100644 src/prism_challenge/ingestion.py create mode 100644 tests/test_prism_audit_effective_tier.py create mode 100644 tests/test_prism_result_ingestion.py diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index 6703e7d..53e4b87 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI, Header, HTTPException, Request, status +from .audit import audit_sampler_from_config from .auth import authenticate_internal, authenticate_validator from .config import PrismSettings, settings from .coordination import list_pending_prism_work_units, work_unit_to_payload @@ -18,6 +19,7 @@ HuggingFaceCheckpointPublisher, ) from .evaluator.interface import PrismContext +from .ingestion import ResultIngestionError, ingest_work_unit_result from .models import ( SubmissionCreate, SubmissionResponse, @@ -156,6 +158,63 @@ async def work_units() -> dict[str, object]: "work_units": [work_unit_to_payload(unit) for unit in units], } + @app.post( + "/internal/v1/work_units/result", dependencies=[Depends(authenticate_internal)] + ) + async def work_unit_result(request: Request) -> dict[str, object]: + """Accept a base-reconciled worker result (``{work_unit_id, submission_ref, result}``). + + The base master forwards exactly one accepted (R=2-reconciled) result here after reconciling + the replicas' manifest hashes (architecture.md 3.3). The ExecutionProof is verified BEFORE + anything is scored: a missing/malformed proof (VAL-PRISM-018) or a tampered manifest / a + forged signature (VAL-PRISM-007) is rejected 422 with a distinguishable reason and never + finalized (the unit stays eligible for retry). A verified result is finalized idempotently: + a duplicate delivery is a no-op and a conflicting delivery for an already-accepted unit is + refused 409 so the stored score/leaderboard is never mutated (VAL-PRISM-017). The claimed + tier is downgraded to its verified effective tier for audit sampling (VAL-PRISM-019). + Disabled with the worker plane off (404) so it is inert in legacy deployments. + """ + if not app_settings.worker_plane.enabled: + raise HTTPException(status.HTTP_404_NOT_FOUND, "worker plane disabled") + try: + payload = json.loads(await request.body()) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, "invalid JSON result body" + ) from exc + if not isinstance(payload, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "result body must be an object") + work_unit_id = payload.get("work_unit_id") + submission_ref = payload.get("submission_ref") + result = payload.get("result") + if not isinstance(work_unit_id, str) or not work_unit_id: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "work_unit_id is required") + if not isinstance(submission_ref, str): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "submission_ref is required") + if not isinstance(result, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "result must be an object") + sampler = audit_sampler_from_config(app_settings.worker_plane) + try: + outcome = await ingest_work_unit_result( + worker=worker, + work_unit_id=work_unit_id, + submission_ref=submission_ref, + result=result, + pinned_image_digest=app_settings.worker_plane.pinned_image_digest, + audit_sampler=sampler, + ) + except ResultIngestionError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + {"code": exc.reason, "detail": str(exc)}, + ) from exc + if outcome.status == "conflict": + raise HTTPException( + status.HTTP_409_CONFLICT, + {"code": outcome.reason, "detail": "conflicting result for finalized unit"}, + ) + return outcome.to_response() + @app.post( "/internal/v1/bridge/submissions", response_model=SubmissionResponse, diff --git a/src/prism_challenge/audit.py b/src/prism_challenge/audit.py new file mode 100644 index 0000000..8f7bc51 --- /dev/null +++ b/src/prism_challenge/audit.py @@ -0,0 +1,148 @@ +"""Tier verification + probabilistic audit sampling (architecture.md 3.4/3.5). + +The proof tier a worker CLAIMS is an input to the audit rate, so it must never be trusted verbatim: +a claim is only worth its tier if the backing metadata is verifiable. :func:`effective_tier` maps a +proof's CLAIMED tier onto the EFFECTIVE tier the audit scheduler actually consumes: + +* claimed tier 2 -> effective 2 iff a populated attestation payload is present, else effective 0 + (a tier-2 claim with a null/empty attestation is downgraded straight to 0, never to 1); +* claimed tier 1 -> effective 1 iff the proof's ``image_digest`` equals the configured pinned + evaluator/worker digest, else effective 0; +* claimed tier 0 (or any unknown tier) -> effective 0. + +:class:`AuditSampler` then samples finalized results at the per-tier rate of their EFFECTIVE tier. +Sampling is deterministic in the sampler's ``seed`` and the per-result key, so the same seed +reproduces the same sample set (VAL-PRISM-011) and a downgraded claim is audited at its lower +(effective) rate rather than the rate it dishonestly claimed (VAL-PRISM-019). +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +from .config import WorkerPlaneConfig +from .proof import ExecutionProof, has_attestation + +#: The three proof tiers the audit rate is keyed on (architecture.md 3.4). +TIER_0, TIER_1, TIER_2 = 0, 1, 2 + + +def effective_tier(proof: ExecutionProof, *, pinned_image_digest: str | None = None) -> int: + """Return the VERIFIED tier for ``proof`` (never higher than what the backing metadata proves). + + A claimed tier is honoured only when its backing is verifiable; otherwise it is downgraded to + tier 0 (architecture.md 3.4; VAL-PRISM-019). A claimed tier 2 with an unverifiable attestation + downgrades to 0 (not 1), even if it also carries a matching image digest. + """ + + claimed = int(proof.tier) + if claimed <= TIER_0: + return TIER_0 + if claimed == TIER_1: + matches = bool(pinned_image_digest) and proof.image_digest == pinned_image_digest + return TIER_1 if matches else TIER_0 + if claimed == TIER_2: + return TIER_2 if has_attestation(proof.attestation) else TIER_0 + # An out-of-range/unknown claimed tier is not verifiable -> conservative tier 0. + return TIER_0 + + +def is_tier_downgraded(proof: ExecutionProof, *, pinned_image_digest: str | None = None) -> bool: + """Whether ``proof``'s claimed tier is higher than its verified effective tier.""" + + return int(proof.tier) != effective_tier(proof, pinned_image_digest=pinned_image_digest) + + +@dataclass(frozen=True) +class AuditDecision: + """The outcome of applying the audit sampler to one finalized result.""" + + work_unit_id: str + claimed_tier: int + effective_tier: int + sampled: bool + + @property + def downgraded(self) -> bool: + return self.claimed_tier != self.effective_tier + + +@dataclass(frozen=True) +class AuditSampler: + """Deterministic, per-tier probabilistic sampler over finalized results (architecture.md 3.4). + + The sampled fraction of each tier converges to its configured rate; a rate of ``0.0`` yields + exactly zero samples for that tier and ``>= 1.0`` samples every one. Sampling is a pure function + of ``seed`` and the per-result key, so it is reproducible and order-insensitive. + """ + + audit_rate_tier0: float = 0.10 + audit_rate_tier1: float = 0.05 + audit_rate_tier2: float = 0.02 + seed: int = 0 + + def rate_for_tier(self, tier: int) -> float: + """The configured audit rate for an EFFECTIVE tier (unknown tiers fall back to tier 0).""" + + if tier == TIER_2: + return self.audit_rate_tier2 + if tier == TIER_1: + return self.audit_rate_tier1 + return self.audit_rate_tier0 + + def should_sample(self, *, work_unit_id: str, effective_tier: int) -> bool: + """Whether a result of ``effective_tier`` is sampled for audit (deterministic in seed).""" + + rate = self.rate_for_tier(effective_tier) + if rate <= 0.0: + return False + if rate >= 1.0: + return True + return self._uniform(work_unit_id) < rate + + def decide( + self, + *, + work_unit_id: str, + proof: ExecutionProof, + pinned_image_digest: str | None = None, + ) -> AuditDecision: + """Verify ``proof``'s tier and decide whether it is sampled at its EFFECTIVE rate.""" + + tier = effective_tier(proof, pinned_image_digest=pinned_image_digest) + return AuditDecision( + work_unit_id=work_unit_id, + claimed_tier=int(proof.tier), + effective_tier=tier, + sampled=self.should_sample(work_unit_id=work_unit_id, effective_tier=tier), + ) + + def _uniform(self, key: str) -> float: + """A deterministic uniform draw in ``[0, 1)`` keyed on ``(seed, key)``.""" + + digest = hashlib.sha256(f"{self.seed}:{key}".encode()).digest() + return int.from_bytes(digest[:8], "big") / float(1 << 64) + + +def audit_sampler_from_config(worker_plane: WorkerPlaneConfig, *, seed: int = 0) -> AuditSampler: + """Build an :class:`AuditSampler` from the prism ``worker_plane`` audit-rate config.""" + + return AuditSampler( + audit_rate_tier0=worker_plane.audit_rate_tier0, + audit_rate_tier1=worker_plane.audit_rate_tier1, + audit_rate_tier2=worker_plane.audit_rate_tier2, + seed=seed, + ) + + +__all__ = [ + "TIER_0", + "TIER_1", + "TIER_2", + "AuditDecision", + "AuditSampler", + "audit_sampler_from_config", + "effective_tier", + "is_tier_downgraded", +] diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index 922d6b4..ea0c5bf 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -27,6 +27,11 @@ class WorkerPlaneConfig(BaseModel): # master-side secret. Unset -> prism emits no signed proof (the base worker plane may still # stamp a tier-0 proof from the manifest hash). signing_key: str | None = Field(default=None, repr=False) + # Pinned evaluator/worker image digest (``sha256:<64hex>``) a claimed tier-1 proof is checked + # against at ingestion: a tier-1 claim whose ``image_digest`` does not match this value is not + # verifiable, so its EFFECTIVE tier is downgraded to 0 for audit sampling (architecture.md 3.4; + # VAL-PRISM-019). Unset -> no tier-1 claim is verifiable, so every tier-1 claim downgrades to 0. + pinned_image_digest: str | None = Field(default=None) class PrismSettings(ChallengeSettings): diff --git a/src/prism_challenge/db.py b/src/prism_challenge/db.py index d901c2d..35919f7 100644 --- a/src/prism_challenge/db.py +++ b/src/prism_challenge/db.py @@ -178,6 +178,12 @@ "effective_from TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1);" "CREATE INDEX IF NOT EXISTS idx_runtime_config_active " "ON runtime_config(config_key, enabled, effective_from, updated_at);" + "CREATE TABLE IF NOT EXISTS work_unit_results (" + "work_unit_id TEXT PRIMARY KEY, submission_id TEXT NOT NULL, manifest_sha256 TEXT NOT NULL," + "claimed_tier INTEGER NOT NULL, effective_tier INTEGER NOT NULL," + "tier_downgraded INTEGER NOT NULL DEFAULT 0, worker_pubkey TEXT, accepted_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_work_unit_results_submission " + "ON work_unit_results(submission_id);" ) @@ -304,6 +310,16 @@ async def _run_migrations(conn: aiosqlite.Connection) -> None: "architecture_id TEXT PRIMARY KEY, content TEXT, model TEXT," "source_submission_id TEXT, generated_at TEXT NOT NULL);" ) + await conn.executescript( + "CREATE TABLE IF NOT EXISTS work_unit_results (" + "work_unit_id TEXT PRIMARY KEY, submission_id TEXT NOT NULL," + "manifest_sha256 TEXT NOT NULL," + "claimed_tier INTEGER NOT NULL, effective_tier INTEGER NOT NULL," + "tier_downgraded INTEGER NOT NULL DEFAULT 0, worker_pubkey TEXT," + "accepted_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_work_unit_results_submission " + "ON work_unit_results(submission_id);" + ) await conn.executescript( "CREATE TABLE IF NOT EXISTS gpu_leases (" "id TEXT PRIMARY KEY, submission_id TEXT NOT NULL, job_id TEXT, target_id TEXT," diff --git a/src/prism_challenge/ingestion.py b/src/prism_challenge/ingestion.py new file mode 100644 index 0000000..cb90a52 --- /dev/null +++ b/src/prism_challenge/ingestion.py @@ -0,0 +1,299 @@ +"""Base->prism result ingestion: verify the ExecutionProof, then finalize idempotently. + +This is the prism half of the base worker-plane accept path (architecture.md 3.3/3.5). After the +base master reconciles a gpu work unit's R=2 replicas it forwards the accepted result here (the +``HttpChallengeResultForwarder`` counterpart) with the pinned body ``{work_unit_id, submission_ref, +result}``. Before anything is scored, the forwarded :class:`~prism_challenge.proof.ExecutionProof` +is verified: + +* **shape** (VAL-PRISM-018): the envelope must exist, be ``version == 1``, carry a 64-char + lowercase-hex ``manifest_sha256`` and a ``worker_signature`` with both ``worker_pubkey`` and + ``sig``; each failure carries a distinguishable reason and NOTHING is scored; +* **integrity** (VAL-PRISM-007): the sr25519 signature must verify against the pinned message for + this unit, and -- when the run manifest is forwarded -- its content must hash back to the signed + ``manifest_sha256`` (a tampered manifest or a forged digest is rejected); +* **tier** (VAL-PRISM-019): the claimed tier is downgraded to its verified EFFECTIVE tier and the + downgrade recorded, so the audit scheduler samples at the honest rate. + +Only a fully verified result is finalized, and finalization is idempotent: the first accepted +delivery finalizes the submission through the CAS-guarded :meth:`PrismWorker.process_submission` +path (a no-op when the executing worker already finalized it against a shared store), while a +duplicate delivery of the same manifest is a no-op and a CONFLICTING delivery (a different +``manifest_sha256`` for an already-accepted unit) is rejected -- never overwriting the stored score +(VAL-PRISM-017). +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +from .audit import AuditSampler, effective_tier +from .auth import verify_hotkey_signature +from .proof import ( + EXECUTION_PROOF_VERSION, + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + ExecutionProof, + compute_manifest_sha256, + verify_execution_proof, +) +from .queue import PrismWorker + +logger = logging.getLogger(__name__) + +#: A verified 64-char lowercase-hex manifest digest (VAL-PRISM-018). +_MANIFEST_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + +SignatureVerifier = Callable[[str, bytes, str], bool] + + +def _as_int(value: object, default: int) -> int: + """Coerce a persisted (sqlite ``object``-typed) tier column back to ``int``.""" + + if isinstance(value, int): + return value + if isinstance(value, str): + try: + return int(value) + except ValueError: + return default + return default + + +class ResultIngestionError(Exception): + """A forwarded result is rejected before scoring; ``reason`` is a stable machine code. + + Reason codes (all distinct from plausibility/scoring failure modes): + + * ``result_malformed`` -- the ``result`` field is not an object; + * ``proof_missing`` -- no ExecutionProof envelope in the result (VAL-PRISM-018a); + * ``proof_bad_version`` -- ``version`` is not ``1`` (VAL-PRISM-018b); + * ``proof_bad_manifest_hash`` -- ``manifest_sha256`` is not 64-char lowercase hex + (VAL-PRISM-018c); + * ``proof_missing_signature`` -- ``worker_signature`` lacks ``worker_pubkey`` or ``sig`` + (VAL-PRISM-018d); + * ``proof_malformed`` -- the envelope fails to parse into an :class:`ExecutionProof`; + * ``manifest_tampered`` -- the forwarded manifest does not hash to ``manifest_sha256`` + (VAL-PRISM-007a); + * ``signature_invalid`` -- the worker signature does not verify for this unit + (VAL-PRISM-007b/c). + """ + + def __init__(self, reason: str, message: str = "") -> None: + self.reason = reason + super().__init__(message or reason) + + +@dataclass(frozen=True) +class IngestionOutcome: + """The observable outcome of ingesting one forwarded result.""" + + status: str # "accepted" | "conflict" + work_unit_id: str + submission_id: str + claimed_tier: int + effective_tier: int + tier_downgraded: bool + idempotent: bool + finalized: bool + submission_status: str | None = None + audit_sampled: bool | None = None + reason: str | None = None + + def to_response(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "status": self.status, + "work_unit_id": self.work_unit_id, + "submission_id": self.submission_id, + "claimed_tier": self.claimed_tier, + "effective_tier": self.effective_tier, + "tier_downgraded": self.tier_downgraded, + "idempotent": self.idempotent, + "finalized": self.finalized, + "submission_status": self.submission_status, + } + if self.audit_sampled is not None: + payload["audit_sampled"] = self.audit_sampled + if self.reason is not None: + payload["reason"] = self.reason + return payload + + +def parse_execution_proof(result: Mapping[str, Any]) -> ExecutionProof: + """Validate the raw proof envelope shape and return a typed :class:`ExecutionProof`. + + Rejects (before any typed coercion so the reason is precise) a missing envelope, a wrong + version, a non-64-hex manifest hash, or a signature block missing either field (VAL-PRISM-018). + """ + + raw = result.get(PROOF_PAYLOAD_KEY) + if raw is None: + raise ResultIngestionError("proof_missing", "result has no execution_proof envelope") + if not isinstance(raw, Mapping): + raise ResultIngestionError("proof_missing", "execution_proof must be an object") + if raw.get("version") != EXECUTION_PROOF_VERSION: + raise ResultIngestionError( + "proof_bad_version", f"unsupported execution_proof version {raw.get('version')!r}" + ) + manifest_sha256 = raw.get("manifest_sha256") + if not isinstance(manifest_sha256, str) or not _MANIFEST_SHA256_RE.fullmatch(manifest_sha256): + raise ResultIngestionError( + "proof_bad_manifest_hash", "manifest_sha256 must be 64-char lowercase hex" + ) + signature = raw.get("worker_signature") + if ( + not isinstance(signature, Mapping) + or not signature.get("worker_pubkey") + or not signature.get("sig") + ): + raise ResultIngestionError( + "proof_missing_signature", "worker_signature must include worker_pubkey and sig" + ) + try: + return ExecutionProof.model_validate(dict(raw)) + except Exception as exc: # noqa: BLE001 - normalise any pydantic error to a stable reason + raise ResultIngestionError("proof_malformed", str(exc)) from exc + + +def verify_proof_integrity( + proof: ExecutionProof, + *, + unit_id: str, + manifest: Mapping[str, Any] | None = None, + verify: SignatureVerifier = verify_hotkey_signature, +) -> None: + """Reject a tampered manifest or a forged/invalid signature for ``unit_id`` (VAL-PRISM-007). + + When the run manifest is forwarded, its canonical content MUST hash to the signed + ``manifest_sha256`` (catches a manifest mutated after signing). The sr25519 signature MUST then + verify against the worker pubkey over the pinned ``sha256(manifest_sha256:unit_id)`` message + (catches a rewritten digest whose signature was not re-issued, or corrupted signature bytes). + """ + + if manifest is not None and compute_manifest_sha256(manifest) != proof.manifest_sha256: + raise ResultIngestionError( + "manifest_tampered", "forwarded manifest does not hash to manifest_sha256" + ) + if not verify_execution_proof(proof, unit_id=unit_id, verify=verify): + raise ResultIngestionError( + "signature_invalid", "worker signature does not verify for this unit" + ) + + +async def ingest_work_unit_result( + *, + worker: PrismWorker, + work_unit_id: str, + submission_ref: str, + result: Mapping[str, Any], + pinned_image_digest: str | None = None, + audit_sampler: AuditSampler | None = None, + verify: SignatureVerifier = verify_hotkey_signature, +) -> IngestionOutcome: + """Verify a forwarded worker result and finalize the submission idempotently. + + ``work_unit_id`` is prism's stable unit id (``== submission_id``). Verification (shape -> + integrity) runs BEFORE any scoring; a rejected result raises :class:`ResultIngestionError` and + leaves the submission untouched (eligible for retry). A verified first delivery finalizes via + the CAS-guarded worker path; a duplicate is an idempotent no-op and a conflicting redelivery for + an already-accepted unit is refused so the stored score/leaderboard is never mutated. + """ + + if not isinstance(result, Mapping): + raise ResultIngestionError("result_malformed", "result must be an object") + + proof = parse_execution_proof(result) + raw_manifest = result.get(MANIFEST_PAYLOAD_KEY) + manifest = raw_manifest if isinstance(raw_manifest, Mapping) else None + verify_proof_integrity(proof, unit_id=work_unit_id, manifest=manifest, verify=verify) + + tier = effective_tier(proof, pinned_image_digest=pinned_image_digest) + claimed_tier = int(proof.tier) + downgraded = claimed_tier != tier + submission_id = work_unit_id + repository = worker.repository + + existing = await repository.get_work_unit_result(work_unit_id) + if existing is not None: + recorded_hash = str(existing.get("manifest_sha256")) + if recorded_hash == proof.manifest_sha256: + return IngestionOutcome( + status="accepted", + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=_as_int(existing.get("claimed_tier"), claimed_tier), + effective_tier=_as_int(existing.get("effective_tier"), tier), + tier_downgraded=bool(existing.get("tier_downgraded", downgraded)), + idempotent=True, + finalized=False, + submission_status=await repository.submission_status(submission_id), + ) + logger.warning( + "rejecting conflicting result delivery for finalized work unit %s " + "(delivered %s != accepted %s); stored score left untouched", + work_unit_id, + proof.manifest_sha256, + recorded_hash, + ) + return IngestionOutcome( + status="conflict", + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=claimed_tier, + effective_tier=tier, + tier_downgraded=downgraded, + idempotent=False, + finalized=False, + submission_status=await repository.submission_status(submission_id), + reason="manifest_conflict", + ) + + if downgraded: + logger.warning( + "downgrading unverifiable tier claim for work unit %s: claimed %d -> effective %d", + work_unit_id, + claimed_tier, + tier, + ) + + result_id = await worker.process_submission(submission_id) + submission_status = await repository.submission_status(submission_id) + await repository.record_work_unit_result( + work_unit_id=work_unit_id, + submission_id=submission_id, + manifest_sha256=proof.manifest_sha256, + claimed_tier=claimed_tier, + effective_tier=tier, + tier_downgraded=downgraded, + worker_pubkey=proof.worker_signature.worker_pubkey, + ) + audit_sampled: bool | None = None + if audit_sampler is not None: + audit_sampled = audit_sampler.should_sample( + work_unit_id=work_unit_id, effective_tier=tier + ) + return IngestionOutcome( + status="accepted", + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=claimed_tier, + effective_tier=tier, + tier_downgraded=downgraded, + idempotent=False, + finalized=result_id is not None, + submission_status=submission_status, + audit_sampled=audit_sampled, + ) + + +__all__ = [ + "IngestionOutcome", + "ResultIngestionError", + "ingest_work_unit_result", + "parse_execution_proof", + "verify_proof_integrity", +] diff --git a/src/prism_challenge/proof.py b/src/prism_challenge/proof.py index d6df9d3..9864bcd 100644 --- a/src/prism_challenge/proof.py +++ b/src/prism_challenge/proof.py @@ -43,6 +43,11 @@ #: plane key so a proof prism emits is passed through unchanged by the base ``WorkerProofExecutor``. PROOF_PAYLOAD_KEY = "execution_proof" +#: Result-payload key carrying the canonical run manifest (the full ``prism_run_manifest.v2`` dict) +#: so the accepting side can recompute ``manifest_sha256`` from the FORWARDED content and reject a +#: tampered manifest whose bytes no longer hash to the signed digest (VAL-PRISM-007). +MANIFEST_PAYLOAD_KEY = "run_manifest" + # Non-secret provider env vars the worker agent injects for the tier-1/2 fields (architecture 3.5). PROVIDER_NAME_ENV = "PRISM_PROVIDER_NAME" EXECUTOR_ID_ENV = "PRISM_EXECUTOR_ID" @@ -360,6 +365,7 @@ def _clean(value: Any) -> str | None: "IMAGE_DIGEST_ENV", "MINER_HOTKEY_ENV", "POD_ID_ENV", + "MANIFEST_PAYLOAD_KEY", "PROOF_PAYLOAD_KEY", "PROVIDER_ENV_KEYS", "PROVIDER_NAME_ENV", diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index e68f6c8..c04e52c 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -1034,6 +1034,57 @@ async def submission_status(self, submission_id: str) -> str | None: return None return str(row_list[0]["status"]) + async def get_work_unit_result(self, work_unit_id: str) -> dict[str, object] | None: + """Return the accepted worker-plane result recorded for ``work_unit_id`` (else ``None``). + + The row is the idempotency/conflict key for base->prism result ingestion: a redelivery of + the same ``manifest_sha256`` is a no-op, a different one for an already-accepted unit is a + conflict, and the persisted claimed/effective tier is the audit-sampling record. + """ + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT * FROM work_unit_results WHERE work_unit_id=?", + (work_unit_id,), + ) + row_list = list(rows) + if not row_list: + return None + return dict(cast(Any, row_list[0])) + + async def record_work_unit_result( + self, + *, + work_unit_id: str, + submission_id: str, + manifest_sha256: str, + claimed_tier: int, + effective_tier: int, + tier_downgraded: bool, + worker_pubkey: str | None, + ) -> None: + """Record the accepted worker-plane result for ``work_unit_id`` (first accept wins). + + ``INSERT OR IGNORE`` keeps the FIRST accepted delivery authoritative: a later same-manifest + redelivery does not rewrite it and a conflicting one is refused upstream, so the accepted + digest + verified tier are stable for idempotency, conflict detection and audit sampling. + """ + async with self.database.connect() as conn: + await conn.execute( + "INSERT OR IGNORE INTO work_unit_results(" + "work_unit_id, submission_id, manifest_sha256, claimed_tier, effective_tier," + "tier_downgraded, worker_pubkey, accepted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + work_unit_id, + submission_id, + manifest_sha256, + int(claimed_tier), + int(effective_tier), + 1 if tier_downgraded else 0, + worker_pubkey, + now_iso(), + ), + ) + async def container_job_attempt_count(self, submission_id: str, level: str) -> int: async with self.database.connect() as conn: rows = await conn.execute_fetchall( diff --git a/src/prism_challenge/validator_dispatch.py b/src/prism_challenge/validator_dispatch.py index 15a6b19..806eb6c 100644 --- a/src/prism_challenge/validator_dispatch.py +++ b/src/prism_challenge/validator_dispatch.py @@ -31,7 +31,7 @@ from .config import PrismSettings from .config import settings as default_settings from .evaluator.checkpoint_publisher import CheckpointPublisher -from .proof import PROOF_PAYLOAD_KEY +from .proof import MANIFEST_PAYLOAD_KEY, PROOF_PAYLOAD_KEY from .validator_executor import run_validator_cycle CHALLENGE_SLUG = "prism" @@ -89,10 +89,14 @@ async def dispatch_assignment( } # Emit the ExecutionProof IN the work-unit result payload at successful finalization # (architecture.md 3.4; VAL-PRISM-001). Absent when the worker plane is off or the unit did not - # freshly finalize. + # freshly finalize. The backing run manifest is forwarded alongside so the accepting plane can + # recompute + verify the signed digest and reject a tampered manifest (VAL-PRISM-007). proof = summary.execution_proofs.get(work_unit_id) if proof is not None: result[PROOF_PAYLOAD_KEY] = proof + manifest = summary.execution_manifests.get(work_unit_id) + if manifest is not None: + result[MANIFEST_PAYLOAD_KEY] = manifest return result diff --git a/src/prism_challenge/validator_executor.py b/src/prism_challenge/validator_executor.py index c4f6ffb..d092cdf 100644 --- a/src/prism_challenge/validator_executor.py +++ b/src/prism_challenge/validator_executor.py @@ -14,10 +14,12 @@ from __future__ import annotations +import json import logging import os from collections.abc import Iterable, Mapping from dataclasses import dataclass, field +from pathlib import Path from typing import Any from .coordination import ( @@ -55,6 +57,9 @@ class PrismWorkUnitExecution: #: The serialized ExecutionProof emitted for this unit's result payload, when the worker plane #: is enabled and a fresh successful finalization produced a manifest to bind (else ``None``). execution_proof: dict[str, Any] | None = None + #: The canonical run manifest whose bytes back ``execution_proof.manifest_sha256``, forwarded so + #: the accepting plane can reject a tampered manifest (VAL-PRISM-007); ``None`` when no proof. + execution_manifest: dict[str, Any] | None = None @dataclass(frozen=True) @@ -67,6 +72,8 @@ class PrismValidatorCycleSummary: completed_submissions: tuple[str, ...] #: ExecutionProofs emitted this cycle, keyed by ``work_unit_id`` (empty when the plane is off). execution_proofs: dict[str, dict[str, Any]] = field(default_factory=dict) + #: Run manifests backing the emitted proofs, keyed by ``work_unit_id`` (empty when off). + execution_manifests: dict[str, dict[str, Any]] = field(default_factory=dict) async def execute_work_unit( @@ -95,8 +102,9 @@ async def execute_work_unit( executed = result_id is not None status = await worker.repository.submission_status(unit.submission_id) execution_proof: dict[str, Any] | None = None + execution_manifest: dict[str, Any] | None = None if executed and status == "completed": - execution_proof = await _emit_execution_proof( + execution_proof, execution_manifest = await _emit_execution_proof( worker, unit, signer=proof_signer, env=proof_env ) return PrismWorkUnitExecution( @@ -106,6 +114,7 @@ async def execute_work_unit( executed=executed, posted=executed, execution_proof=execution_proof, + execution_manifest=execution_manifest, ) @@ -115,16 +124,17 @@ async def _emit_execution_proof( *, signer: WorkerSigner | None, env: Mapping[str, str] | None, -) -> dict[str, Any] | None: - """Build the ExecutionProof for a freshly finalized unit, or ``None`` when not applicable. +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Build the ExecutionProof + its backing manifest for a finalized unit (else ``(None, None)``). Gated on ``worker_plane.enabled``. The manifest hash is taken from the exact on-disk bytes of the run's ``prism_run_manifest.v2.json``; the provenance comes ONLY from the non-secret provider - env allowlist. Held-out split config and LLM keys are never read (VAL-PRISM-008). + env allowlist. Held-out split config and LLM keys are never read (VAL-PRISM-008). The manifest + content is returned with the proof so the accepting plane can recompute + verify the digest. """ if not worker.settings.worker_plane.enabled: - return None + return None, None resolved_signer = _resolve_proof_signer(worker, signer) if resolved_signer is None: logger.warning( @@ -132,19 +142,29 @@ async def _emit_execution_proof( "for work_unit=%s", unit.work_unit_id, ) - return None + return None, None manifest_path = await worker.repository.latest_run_manifest_path( unit.submission_id, worker.execution_backend ) if not manifest_path: - return None + return None, None proof = build_execution_proof_from_manifest( signer=resolved_signer, unit_id=unit.work_unit_id, manifest_path=manifest_path, env=os.environ if env is None else env, ) - return proof.model_dump(mode="json") + return proof.model_dump(mode="json"), _load_manifest(manifest_path) + + +def _load_manifest(manifest_path: str) -> dict[str, Any] | None: + """Load the run manifest JSON from ``manifest_path`` (``None`` when it cannot be parsed).""" + + try: + parsed = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return parsed if isinstance(parsed, dict) else None def _resolve_proof_signer( @@ -194,6 +214,7 @@ async def run_validator_cycle( skipped = 0 completed: list[str] = [] execution_proofs: dict[str, dict[str, Any]] = {} + execution_manifests: dict[str, dict[str, Any]] = {} for unit in pulled: outcome = await execute_work_unit( worker, unit, proof_signer=proof_signer, proof_env=proof_env @@ -206,10 +227,13 @@ async def run_validator_cycle( completed.append(outcome.submission_id) if outcome.execution_proof is not None: execution_proofs[outcome.work_unit_id] = outcome.execution_proof + if outcome.execution_manifest is not None: + execution_manifests[outcome.work_unit_id] = outcome.execution_manifest return PrismValidatorCycleSummary( pulled=len(pulled), executed=executed, skipped=skipped, completed_submissions=tuple(completed), execution_proofs=execution_proofs, + execution_manifests=execution_manifests, ) diff --git a/tests/test_prism_audit_effective_tier.py b/tests/test_prism_audit_effective_tier.py new file mode 100644 index 0000000..beea2f8 --- /dev/null +++ b/tests/test_prism_audit_effective_tier.py @@ -0,0 +1,139 @@ +"""Tier verification + effective-tier audit sampling (VAL-PRISM-019). + +A worker's CLAIMED proof tier is only honoured when its backing metadata is verifiable; otherwise +the EFFECTIVE tier used for audit sampling is downgraded. These tests pin the downgrade rules and +show that seeded sampling statistics follow the EFFECTIVE tier, not the dishonest claim. Offline, +pure functions (no GPU, no DB). +""" + +from __future__ import annotations + +from prism_challenge.audit import ( + AuditSampler, + audit_sampler_from_config, + effective_tier, + is_tier_downgraded, +) +from prism_challenge.config import WorkerPlaneConfig +from prism_challenge.proof import ExecutionProof, ProviderInfo, WorkerSignature + +PINNED = "sha256:" + "a" * 64 +OTHER = "sha256:" + "b" * 64 + + +def _proof( + *, + tier: int, + image_digest: str | None = None, + attestation: dict[str, object] | None = None, +) -> ExecutionProof: + return ExecutionProof( + version=1, + tier=tier, + manifest_sha256="c" * 64, + image_digest=image_digest, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + attestation=attestation, + ) + + +# --- Downgrade rules (VAL-PRISM-019 a/b) -------------------------------------------------------- + + +def test_tier2_claim_downgrades_without_valid_attestation() -> None: + valid = _proof(tier=2, attestation={"tdx_quote_b64": "abc"}) + null_attest = _proof(tier=2, attestation=None) + empty_attest = _proof(tier=2, attestation={"unrelated": "x"}) + + assert effective_tier(valid, pinned_image_digest=PINNED) == 2 + assert is_tier_downgraded(valid, pinned_image_digest=PINNED) is False + # A tier-2 claim with a null / keyless attestation is effective tier 0 (never 2, never 1) -- + # even when it also carries a matching digest. + assert effective_tier(null_attest, pinned_image_digest=PINNED) == 0 + assert effective_tier(empty_attest, pinned_image_digest=PINNED) == 0 + with_digest = _proof(tier=2, image_digest=PINNED, attestation=None) + assert effective_tier(with_digest, pinned_image_digest=PINNED) == 0 + assert is_tier_downgraded(with_digest, pinned_image_digest=PINNED) is True + + +def test_tier1_claim_requires_matching_pinned_digest() -> None: + matching = _proof(tier=1, image_digest=PINNED) + mismatched = _proof(tier=1, image_digest=OTHER) + missing = _proof(tier=1, image_digest=None) + + assert effective_tier(matching, pinned_image_digest=PINNED) == 1 + assert is_tier_downgraded(matching, pinned_image_digest=PINNED) is False + assert effective_tier(mismatched, pinned_image_digest=PINNED) == 0 + assert effective_tier(missing, pinned_image_digest=PINNED) == 0 + # With no pinned digest configured, no tier-1 claim is verifiable. + assert effective_tier(matching, pinned_image_digest=None) == 0 + + +def test_tier0_claim_stays_tier0() -> None: + assert effective_tier(_proof(tier=0), pinned_image_digest=PINNED) == 0 + assert is_tier_downgraded(_proof(tier=0), pinned_image_digest=PINNED) is False + + +# --- Sampling follows the EFFECTIVE tier (VAL-PRISM-019 statistical) ----------------------------- + + +def _sampled_fraction(sampler: AuditSampler, proof: ExecutionProof, n: int) -> float: + hits = sum( + sampler.decide( + work_unit_id=f"{proof.tier}-{i}", proof=proof, pinned_image_digest=PINNED + ).sampled + for i in range(n) + ) + return hits / n + + +def test_sampling_statistics_follow_effective_not_claimed_tier() -> None: + sampler = AuditSampler( + audit_rate_tier0=0.10, audit_rate_tier1=0.05, audit_rate_tier2=0.02, seed=1234 + ) + n = 6000 + # 4-sigma binomial bound around each configured rate for this N. + def _bound(p: float) -> float: + return 4.0 * (p * (1.0 - p) / n) ** 0.5 + + honest_t2 = _proof(tier=2, attestation={"gpu_eat_jwt": "jwt"}) + honest_t1 = _proof(tier=1, image_digest=PINNED) + fake_t2 = _proof(tier=2, attestation=None) # effective 0 + fake_t1 = _proof(tier=1, image_digest=OTHER) # effective 0 + + # Honest claims are sampled at their own tier's rate. + assert abs(_sampled_fraction(sampler, honest_t2, n) - 0.02) < _bound(0.02) + assert abs(_sampled_fraction(sampler, honest_t1, n) - 0.05) < _bound(0.05) + # Unverifiable claims are sampled at the EFFECTIVE (tier-0) rate, NOT the lower claimed rate. + assert abs(_sampled_fraction(sampler, fake_t2, n) - 0.10) < _bound(0.10) + assert abs(_sampled_fraction(sampler, fake_t1, n) - 0.10) < _bound(0.10) + + +def test_zero_rate_never_samples_and_seed_is_reproducible() -> None: + zero = AuditSampler(audit_rate_tier0=0.0, audit_rate_tier1=0.0, audit_rate_tier2=0.0, seed=7) + assert all( + not zero.should_sample(work_unit_id=f"u{i}", effective_tier=t) + for i in range(500) + for t in (0, 1, 2) + ) + + a = AuditSampler(audit_rate_tier0=0.10, seed=99) + b = AuditSampler(audit_rate_tier0=0.10, seed=99) + c = AuditSampler(audit_rate_tier0=0.10, seed=100) + ids = [f"unit-{i}" for i in range(300)] + sample_a = [a.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + sample_b = [b.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + sample_c = [c.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + assert sample_a == sample_b # same seed => identical sample set + assert sample_a != sample_c # a different seed shifts the sample set + + +def test_sampler_from_config_uses_configured_rates() -> None: + config = WorkerPlaneConfig( + enabled=True, audit_rate_tier0=0.4, audit_rate_tier1=0.3, audit_rate_tier2=0.2 + ) + sampler = audit_sampler_from_config(config, seed=5) + assert sampler.rate_for_tier(0) == 0.4 + assert sampler.rate_for_tier(1) == 0.3 + assert sampler.rate_for_tier(2) == 0.2 diff --git a/tests/test_prism_result_ingestion.py b/tests/test_prism_result_ingestion.py new file mode 100644 index 0000000..2a4fdfc --- /dev/null +++ b/tests/test_prism_result_ingestion.py @@ -0,0 +1,546 @@ +"""base->prism result ingestion: proof verification + idempotent finalization. + +Covers VAL-PRISM-007 (tampered manifest / forged digest rejected), VAL-PRISM-018 (missing/malformed +ExecutionProof rejected before scoring) and VAL-PRISM-017 (duplicate/late delivery never mutates a +finalized submission). Offline, no GPU: finalization runs through the CPU re-exec seam the other +prism coordination tests use, and proofs are signed with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import io +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.ingestion import ( + ResultIngestionError, + ingest_work_unit_result, + parse_execution_proof, + verify_proof_integrity, +) +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) + +WORKER_KEY = "//WorkerIngest" + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=worker_plane, + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + return { + "schema_version": "prism_run_manifest.v2", + "metrics": {"prequential_bpb": 1.23, "marker": marker}, + "steps": [{"step": 0, "loss": 3.0}, {"step": 1, "loss": 2.0}], + } + + +def _proof_dict(signer, unit_id: str, manifest: dict[str, Any], **overrides: Any) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + payload = proof.model_dump(mode="json") + payload.update(overrides) + return payload + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any] | None) -> dict[str, Any]: + result: dict[str, Any] = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + } + if manifest is not None: + result[MANIFEST_PAYLOAD_KEY] = manifest + return result + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _score(db_path: Path, submission_id: str): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return row[0] if row else None + + +# --- VAL-PRISM-018: missing / malformed ExecutionProof rejected before scoring ------------------- + + +def test_parse_rejects_missing_and_malformed_proof() -> None: + signer = worker_signer_from_key(WORKER_KEY) + manifest = _manifest() + good = _proof_dict(signer, "unit-1", manifest) + + # (a) envelope entirely absent + with pytest.raises(ResultIngestionError) as absent: + parse_execution_proof({}) + assert absent.value.reason == "proof_missing" + + # (b) version != 1 + with pytest.raises(ResultIngestionError) as version: + parse_execution_proof(_result({**good, "version": 2}, manifest)) + assert version.value.reason == "proof_bad_version" + + # (c) manifest_sha256 not 64-char lowercase hex (uppercase, too short, non-hex) + for bad_hash in (good["manifest_sha256"].upper(), "abc", "z" * 64): + with pytest.raises(ResultIngestionError) as bad: + parse_execution_proof(_result({**good, "manifest_sha256": bad_hash}, manifest)) + assert bad.value.reason == "proof_bad_manifest_hash" + + # (d) worker_signature missing worker_pubkey or sig + pubkey = good["worker_signature"]["worker_pubkey"] + for signature in ({"worker_pubkey": pubkey}, {"sig": "0xab"}): + with pytest.raises(ResultIngestionError) as sig: + parse_execution_proof(_result({**good, "worker_signature": signature}, manifest)) + assert sig.value.reason == "proof_missing_signature" + + # A well-formed control parses. + assert parse_execution_proof(_result(good, manifest)).version == 1 + + +async def test_ingestion_rejects_malformed_proof_before_scoring(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + # Each malformed variant is rejected with a distinguishable reason and scores nothing. + variants = { + "proof_missing": {"executed": 1}, + "proof_bad_version": None, # filled below + "proof_bad_manifest_hash": None, + "proof_missing_signature": None, + } + for reason in list(variants): + submission_id = await _seed(app) + manifest = _manifest() + good = _proof_dict(signer, submission_id, manifest) + if reason == "proof_missing": + result = {"executed": 1} + elif reason == "proof_bad_version": + result = _result({**good, "version": 3}, manifest) + elif reason == "proof_bad_manifest_hash": + result = _result({**good, "manifest_sha256": "nothex"}, manifest) + else: + result = _result({**good, "worker_signature": {"worker_pubkey": "x"}}, manifest) + + with pytest.raises(ResultIngestionError) as exc: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=result, + ) + assert exc.value.reason == reason + assert _score(db_path, submission_id) is None + assert await app.state.repository.submission_status(submission_id) == "pending" + + # A well-formed control from the same fixture IS accepted and finalizes a score. + submission_id = await _seed(app) + manifest = _manifest() + good = _proof_dict(signer, submission_id, manifest) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(good, manifest), + ) + assert outcome.status == "accepted" + assert outcome.finalized is True + assert await app.state.repository.submission_status(submission_id) == "completed" + assert _score(db_path, submission_id) is not None + + +# --- VAL-PRISM-007: tampered manifest / forged digest rejected ---------------------------------- + + +def test_verify_rejects_tamper_and_forgery() -> None: + signer = worker_signer_from_key(WORKER_KEY) + unit_id = "unit-7" + manifest = _manifest() + proof = build_execution_proof( + signer=signer, manifest_sha256=compute_manifest_sha256(manifest), unit_id=unit_id + ) + + # A genuine (manifest, proof) pair verifies. + verify_proof_integrity(proof, unit_id=unit_id, manifest=manifest) + + # (a) a manifest mutated after signing no longer hashes to manifest_sha256 + tampered_manifest = {**manifest, "metrics": {**manifest["metrics"], "prequential_bpb": 0.01}} + with pytest.raises(ResultIngestionError) as tamper: + verify_proof_integrity(proof, unit_id=unit_id, manifest=tampered_manifest) + assert tamper.value.reason == "manifest_tampered" + + # (b) manifest_sha256 rewritten to match the tampered manifest, signature NOT re-issued + forged = proof.model_copy( + update={"manifest_sha256": compute_manifest_sha256(tampered_manifest)} + ) + with pytest.raises(ResultIngestionError) as forged_hash: + verify_proof_integrity(forged, unit_id=unit_id, manifest=tampered_manifest) + assert forged_hash.value.reason == "signature_invalid" + + # (c) corrupted signature bytes + corrupt = proof.model_copy( + update={"worker_signature": proof.worker_signature.model_copy(update={"sig": "0x00"})} + ) + with pytest.raises(ResultIngestionError) as corrupt_sig: + verify_proof_integrity(corrupt, unit_id=unit_id, manifest=manifest) + assert corrupt_sig.value.reason == "signature_invalid" + + +async def test_ingestion_rejects_tampered_result_without_scoring(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + for kind in ("tamper", "forged_digest", "corrupt_sig"): + submission_id = await _seed(app) + manifest = _manifest() + proof = _proof_dict(signer, submission_id, manifest) + if kind == "tamper": + result = _result(proof, {**manifest, "metrics": {"prequential_bpb": 9.9}}) + expected = "manifest_tampered" + elif kind == "forged_digest": + tampered = {**manifest, "metrics": {"prequential_bpb": 9.9}} + forged = {**proof, "manifest_sha256": compute_manifest_sha256(tampered)} + result = _result(forged, tampered) + expected = "signature_invalid" + else: + corrupt = {**proof, "worker_signature": {**proof["worker_signature"], "sig": "0x00"}} + result = _result(corrupt, manifest) + expected = "signature_invalid" + + with pytest.raises(ResultIngestionError) as exc: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=result, + ) + assert exc.value.reason == expected + assert _score(db_path, submission_id) is None + assert await app.state.repository.submission_status(submission_id) == "pending" + + +# --- VAL-PRISM-017: duplicate / late delivery never mutates a finalized submission --------------- + + +async def test_duplicate_and_conflicting_delivery_never_mutates_score( + tmp_path, monkeypatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + submission_id = await _seed(app) + manifest = _manifest() + proof = _proof_dict(signer, submission_id, manifest) + + # First delivery finalizes. + first = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert first.status == "accepted" and first.finalized is True + score_after_first = _score(db_path, submission_id) + assert score_after_first is not None + jobs_after_first = _eval_job_count(db_path, submission_id) + + # (a) a SECOND delivery of the SAME result is an idempotent no-op: no re-score, no new eval job. + second = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert second.status == "accepted" and second.idempotent is True and second.finalized is False + assert _score(db_path, submission_id) == pytest.approx(score_after_first) + assert _eval_job_count(db_path, submission_id) == jobs_after_first + + # (b) a CONFLICTING delivery (different, validly-signed manifest) is refused, not applied. + conflict_manifest = _manifest("conflict") + conflict_proof = _proof_dict(signer, submission_id, conflict_manifest) + conflicting = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(conflict_proof, conflict_manifest), + ) + assert conflicting.status == "conflict" and conflicting.reason == "manifest_conflict" + assert _score(db_path, submission_id) == pytest.approx(score_after_first) + assert _eval_job_count(db_path, submission_id) == jobs_after_first + + # (c) the legacy in-process equivalent: process_submission on a finalized submission is a no-op. + assert await app.state.worker.process_submission(submission_id) is None + assert _score(db_path, submission_id) == pytest.approx(score_after_first) + + +def _eval_job_count(db_path: Path, submission_id: str) -> int: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT COUNT(*) FROM eval_jobs WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return int(row[0]) if row else 0 + + +# --- VAL-PRISM-019: unverifiable tier claims are downgraded + recorded at ingestion -------------- + + +async def test_ingestion_records_downgraded_effective_tier(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + submission_id = await _seed(app) + manifest = _manifest() + # A tier-2 claim with a null attestation is unverifiable -> effective tier 0. + proof = _proof_dict(signer, submission_id, manifest, tier=2, attestation=None) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert outcome.claimed_tier == 2 + assert outcome.effective_tier == 0 + assert outcome.tier_downgraded is True + + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT claimed_tier, effective_tier, tier_downgraded FROM work_unit_results " + "WHERE work_unit_id=?", + (submission_id,), + ).fetchone() + finally: + conn.close() + assert row == (2, 0, 1) + + +# --- HTTP route body contract + status codes (VAL-PRISM-017/018) --------------------------------- + + +def test_result_route_body_contract(tmp_path, monkeypatch) -> None: + from fastapi.testclient import TestClient + + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + signer = worker_signer_from_key(WORKER_KEY) + headers = {"Authorization": "Bearer secret"} + + with TestClient(create_app(settings)) as client: + seed = client.post( + "/internal/v1/bridge/submissions", + content=_bundle().encode(), + headers={ + "Authorization": "Bearer secret", + "X-Base-Verified-Hotkey": "hk-owner", + "X-Submission-Filename": "project.zip", + "Content-Type": "application/octet-stream", + }, + ) + assert seed.status_code == 200, seed.text + submission_id = seed.json()["id"] + + manifest = _manifest() + proof = _proof_dict(signer, submission_id, manifest) + body = { + "work_unit_id": submission_id, + "submission_ref": "hk-owner", + "result": _result(proof, manifest), + } + + # Unauthenticated is rejected. + assert client.post("/internal/v1/work_units/result", json=body).status_code == 401 + + # A well-formed forwarded result is accepted (exact base body contract). + accept = client.post("/internal/v1/work_units/result", json=body, headers=headers) + assert accept.status_code == 200, accept.text + assert accept.json()["status"] == "accepted" + + # A malformed proof is rejected 422 with a distinguishable reason code. + bad = { + "work_unit_id": submission_id, + "submission_ref": "hk-owner", + "result": _result({**proof, "version": 5}, manifest), + } + rejected = client.post("/internal/v1/work_units/result", json=bad, headers=headers) + assert rejected.status_code == 422 + assert rejected.json()["detail"]["code"] == "proof_bad_version" + + # A conflicting redelivery for the finalized unit is refused 409. + conflict_manifest = _manifest("conflict") + conflict_proof = _proof_dict(signer, submission_id, conflict_manifest) + conflict = client.post( + "/internal/v1/work_units/result", + json={ + "work_unit_id": submission_id, + "submission_ref": "hk-owner", + "result": _result(conflict_proof, conflict_manifest), + }, + headers=headers, + ) + assert conflict.status_code == 409 + assert conflict.json()["detail"]["code"] == "manifest_conflict" + + +def test_result_route_disabled_when_worker_plane_off(tmp_path) -> None: + from fastapi.testclient import TestClient + + settings = _settings(tmp_path, worker_plane=WorkerPlaneConfig(enabled=False)) + with TestClient(create_app(settings)) as client: + resp = client.post( + "/internal/v1/work_units/result", + json={"work_unit_id": "s1", "submission_ref": "hk", "result": {}}, + headers={"Authorization": "Bearer secret"}, + ) + assert resp.status_code == 404 + From d3cd94427b763fbc01116b81c541252f95d7f2c5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:29:15 +0000 Subject: [PATCH 03/14] feat(prism): plausibility gate for worker-reported results (M3 prism-proof) Add a verify-only plausibility module (architecture.md 3.5) that rejects an implausible prism_run_manifest.v2 before scoring and passes a plausible manifest through unchanged: - schema/version + metrics-block validation - step-0 anomaly (initial loss below the from-scratch baseline) - loss-trajectory sanity (final/late online loss at/above ~ln(vocab)) - log/score consistency - wall-clock vs declared budget Wired into the base->prism verify-only ingestion path with a distinct plausibility_* rejection reason (never scored, unit stays retry-eligible); plausible results finalize identically to the legacy path (VAL-PRISM-009/010). --- src/prism_challenge/app.py | 11 +- src/prism_challenge/ingestion.py | 17 +- src/prism_challenge/plausibility.py | 221 ++++++++++++++ tests/test_prism_plausibility.py | 435 ++++++++++++++++++++++++++++ 4 files changed, 680 insertions(+), 4 deletions(-) create mode 100644 src/prism_challenge/plausibility.py create mode 100644 tests/test_prism_plausibility.py diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index 53e4b87..0732c0b 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -24,6 +24,7 @@ SubmissionCreate, SubmissionResponse, ) +from .plausibility import PlausibilityError from .queue import PrismWorker from .repository import PrismRepository from .routes import router @@ -168,7 +169,10 @@ async def work_unit_result(request: Request) -> dict[str, object]: the replicas' manifest hashes (architecture.md 3.3). The ExecutionProof is verified BEFORE anything is scored: a missing/malformed proof (VAL-PRISM-018) or a tampered manifest / a forged signature (VAL-PRISM-007) is rejected 422 with a distinguishable reason and never - finalized (the unit stays eligible for retry). A verified result is finalized idempotently: + finalized (the unit stays eligible for retry). A verified result is then run through the + plausibility gate (architecture.md 3.5; VAL-PRISM-009): an implausible manifest is rejected + 422 with a distinct ``plausibility_*`` reason and never scored, while a plausible manifest + passes through UNCHANGED. A verified, plausible result is finalized idempotently: a duplicate delivery is a no-op and a conflicting delivery for an already-accepted unit is refused 409 so the stored score/leaderboard is never mutated (VAL-PRISM-017). The claimed tier is downgraded to its verified effective tier for audit sampling (VAL-PRISM-019). @@ -208,6 +212,11 @@ async def work_unit_result(request: Request) -> dict[str, object]: status.HTTP_422_UNPROCESSABLE_ENTITY, {"code": exc.reason, "detail": str(exc)}, ) from exc + except PlausibilityError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + {"code": exc.reason, "detail": str(exc)}, + ) from exc if outcome.status == "conflict": raise HTTPException( status.HTTP_409_CONFLICT, diff --git a/src/prism_challenge/ingestion.py b/src/prism_challenge/ingestion.py index cb90a52..7a9df74 100644 --- a/src/prism_challenge/ingestion.py +++ b/src/prism_challenge/ingestion.py @@ -33,6 +33,7 @@ from .audit import AuditSampler, effective_tier from .auth import verify_hotkey_signature +from .plausibility import check_manifest_plausibility from .proof import ( EXECUTION_PROOF_VERSION, MANIFEST_PAYLOAD_KEY, @@ -198,9 +199,13 @@ async def ingest_work_unit_result( ``work_unit_id`` is prism's stable unit id (``== submission_id``). Verification (shape -> integrity) runs BEFORE any scoring; a rejected result raises :class:`ResultIngestionError` and - leaves the submission untouched (eligible for retry). A verified first delivery finalizes via - the CAS-guarded worker path; a duplicate is an idempotent no-op and a conflicting redelivery for - an already-accepted unit is refused so the stored score/leaderboard is never mutated. + leaves the submission untouched (eligible for retry). A verified first delivery is then run + through the plausibility gate (architecture.md 3.5; VAL-PRISM-009): an implausible manifest + raises :class:`~prism_challenge.plausibility.PlausibilityError` (a reason DISTINCT from the + proof-verification reasons) and is never scored, while a plausible manifest passes through + UNCHANGED and finalizes via the CAS-guarded worker path. A duplicate is an idempotent no-op and + a conflicting redelivery for an already-accepted unit is refused so the stored score/leaderboard + is never mutated. """ if not isinstance(result, Mapping): @@ -260,6 +265,12 @@ async def ingest_work_unit_result( tier, ) + if manifest is not None: + check_manifest_plausibility( + manifest, + wall_clock_budget_seconds=float(worker.settings.base_eval_hard_timeout_seconds), + ) + result_id = await worker.process_submission(submission_id) submission_status = await repository.submission_status(submission_id) await repository.record_work_unit_result( diff --git a/src/prism_challenge/plausibility.py b/src/prism_challenge/plausibility.py new file mode 100644 index 0000000..10208d8 --- /dev/null +++ b/src/prism_challenge/plausibility.py @@ -0,0 +1,221 @@ +"""Plausibility gate for worker-reported results (architecture.md 3.5). + +The verify-only validator pipeline rejects a worker-reported run whose ``prism_run_manifest.v2`` +is implausible BEFORE it is ever scored, routing it to the reject/retry path. A plausible manifest +passes through UNCHANGED (this module only READS the manifest; it never mutates the payload or the +score inputs), so a genuine result finalizes to the exact same score as the legacy path. + +The checks, in order (architecture.md 3.5), each firing only when the manifest carries the data it +needs so a sparse-but-honest manifest is never spuriously rejected: + +* **schema/version** -- ``schema_version`` must be ``prism_run_manifest.v2`` and ``metrics`` must be + a well-formed object; +* **step-0 anomaly** -- an initial loss impossibly below the from-scratch baseline (< the existing + ``STEP0_ANOMALY_FRACTION`` of ``~ln(vocab)`` nats) is the smuggled-pretrained-weights signal; +* **loss-trajectory sanity** -- a final/late online loss still sitting at (or above) the random-init + baseline means no learning happened; +* **log/score consistency** -- the logged online-loss metrics must agree with the derived score + block (a self-inconsistent manifest is fabricated); +* **wall-clock vs declared budget** -- a declared wall-clock grossly beyond the budget (or negative) + is implausible. + +Rejection reasons are all namespaced ``plausibility_*`` so they are DISTINCT from the proof- +verification reasons in :mod:`prism_challenge.ingestion` (VAL-PRISM-018 requires the two rejection +classes be distinguishable). +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any + +#: Only the challenge-authored v2 re-execution manifest is scored (schemas.RUN_MANIFEST_V2_...). +MANIFEST_SCHEMA_VERSION = "prism_run_manifest.v2" + +# Kept in sync with ``evaluator/container.py``'s ``STEP0_ANOMALY_FRACTION``: the step-0 forced-init +# loss must sit near the from-scratch baseline (~ln(vocab) nats); an initial loss below this +# fraction of the baseline is the smuggled-pretrained-weights anomaly. +STEP0_ANOMALY_FRACTION = 0.5 +# A learner's late loss must fall meaningfully below the from-scratch baseline; a late loss at or +# above this fraction of the baseline means no learning happened (architecture.md 3.5). +NO_LEARNING_FRACTION = 0.98 +# A declared wall-clock beyond this multiple of the run's wall-clock budget is grossly inconsistent +# (the run would have been stopped at the hard cap long before). +WALL_CLOCK_BUDGET_MULTIPLIER = 2.0 +# Relative tolerance for the logged-vs-derived loss consistency check. +LOSS_CONSISTENCY_REL_TOL = 1e-3 +LOSS_CONSISTENCY_ABS_TOL = 1e-6 + +REASON_SCHEMA_VERSION = "plausibility_schema_version" +REASON_METRICS_MALFORMED = "plausibility_metrics_malformed" +REASON_STEP0_ANOMALY = "plausibility_step0_anomaly" +REASON_LOSS_AT_BASELINE = "plausibility_loss_at_baseline" +REASON_LOSS_INCONSISTENT = "plausibility_loss_inconsistent" +REASON_WALLCLOCK_BUDGET = "plausibility_wallclock_budget" + + +class PlausibilityError(Exception): + """A worker-reported result is implausible and must not be scored. + + ``reason`` is a stable machine code namespaced ``plausibility_*`` so a plausibility rejection is + always distinguishable from a proof-verification rejection (VAL-PRISM-009/018). + """ + + def __init__(self, reason: str, message: str = "") -> None: + self.reason = reason + super().__init__(message or reason) + + +def check_manifest_plausibility( + manifest: Mapping[str, Any], + *, + wall_clock_budget_seconds: float | None = None, +) -> None: + """Raise :class:`PlausibilityError` if ``manifest`` is implausible; return ``None`` otherwise. + + Read-only: the manifest is never mutated, so a plausible result passes through unchanged. Each + numeric check runs only when its inputs are present, so a manifest that simply omits a + trajectory/compute field is not rejected for the absence alone (only the schema/version check is + unconditional). + """ + + if not isinstance(manifest, Mapping): + raise PlausibilityError(REASON_METRICS_MALFORMED, "manifest must be an object") + if manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: + raise PlausibilityError( + REASON_SCHEMA_VERSION, + f"manifest schema_version must be {MANIFEST_SCHEMA_VERSION!r}", + ) + metrics = manifest.get("metrics") + if not isinstance(metrics, Mapping): + raise PlausibilityError(REASON_METRICS_MALFORMED, "manifest metrics block is malformed") + + anti_cheat = manifest.get("anti_cheat") + anti_cheat = anti_cheat if isinstance(anti_cheat, Mapping) else {} + baseline = _opt_float(metrics.get("random_init_baseline_nats")) + online_loss = _loss_series(metrics.get("online_loss")) + step0 = _opt_float(metrics.get("step0_loss")) + if step0 is None and online_loss: + step0 = online_loss[0] + + _check_step0_anomaly(anti_cheat, baseline=baseline, step0=step0) + _check_loss_trajectory(anti_cheat, baseline=baseline, online_loss=online_loss) + _check_log_consistency(manifest, metrics, online_loss=online_loss, step0=step0) + _check_wallclock(manifest, budget_seconds=wall_clock_budget_seconds) + + +def _check_step0_anomaly( + anti_cheat: Mapping[str, Any], *, baseline: float | None, step0: float | None +) -> None: + if anti_cheat.get("step0_anomaly") is True: + raise PlausibilityError(REASON_STEP0_ANOMALY, "manifest flags a step-0 anomaly") + if baseline is not None and step0 is not None and step0 < STEP0_ANOMALY_FRACTION * baseline: + raise PlausibilityError( + REASON_STEP0_ANOMALY, + "step-0 loss is impossibly below the from-scratch baseline", + ) + + +def _check_loss_trajectory( + anti_cheat: Mapping[str, Any], *, baseline: float | None, online_loss: list[float] +) -> None: + if anti_cheat.get("no_learning") is True or anti_cheat.get("zero_forward") is True: + raise PlausibilityError(REASON_LOSS_AT_BASELINE, "manifest flags a no-learning run") + if baseline is None or not online_loss: + return + late = _late_loss(online_loss) + if late >= NO_LEARNING_FRACTION * baseline: + raise PlausibilityError( + REASON_LOSS_AT_BASELINE, + "final/late online loss sits at or above the random-init baseline (no learning)", + ) + + +def _check_log_consistency( + manifest: Mapping[str, Any], + metrics: Mapping[str, Any], + *, + online_loss: list[float], + step0: float | None, +) -> None: + reported_step0 = _opt_float(metrics.get("step0_loss")) + if reported_step0 is not None and online_loss and not _close(reported_step0, online_loss[0]): + raise PlausibilityError( + REASON_LOSS_INCONSISTENT, + "reported step0_loss disagrees with the logged online-loss curve", + ) + metrics_bpb = _opt_float(metrics.get("prequential_bpb")) + score = manifest.get("score") + score_bpb = _opt_float(score.get("prequential_bpb")) if isinstance(score, Mapping) else None + if metrics_bpb is not None and score_bpb is not None and not _close(metrics_bpb, score_bpb): + raise PlausibilityError( + REASON_LOSS_INCONSISTENT, + "score-block bits-per-byte disagrees with the logged metrics", + ) + + +def _check_wallclock(manifest: Mapping[str, Any], *, budget_seconds: float | None) -> None: + compute = manifest.get("compute") + wall_clock = ( + _opt_float(compute.get("wall_clock_seconds")) if isinstance(compute, Mapping) else None + ) + if wall_clock is None: + return + if wall_clock < 0.0: + raise PlausibilityError(REASON_WALLCLOCK_BUDGET, "declared wall-clock is negative") + if ( + budget_seconds is not None + and budget_seconds > 0.0 + and wall_clock > budget_seconds * WALL_CLOCK_BUDGET_MULTIPLIER + ): + raise PlausibilityError( + REASON_WALLCLOCK_BUDGET, + "declared wall-clock is grossly inconsistent with the declared budget", + ) + + +def _loss_series(value: Any) -> list[float]: + if not isinstance(value, (list, tuple)): + return [] + series = [_opt_float(item) for item in value] + return [item for item in series if item is not None] + + +def _late_loss(online_loss: list[float]) -> float: + """The mean of the trailing quarter of the loss curve (the run's converged/"late" loss).""" + + tail = max(1, len(online_loss) // 4) + window = online_loss[-tail:] + return sum(window) / len(window) + + +def _close(left: float, right: float) -> bool: + return math.isclose( + left, right, rel_tol=LOSS_CONSISTENCY_REL_TOL, abs_tol=LOSS_CONSISTENCY_ABS_TOL + ) + + +def _opt_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + number = float(value) + return number if math.isfinite(number) else None + return None + + +__all__ = [ + "MANIFEST_SCHEMA_VERSION", + "NO_LEARNING_FRACTION", + "REASON_LOSS_AT_BASELINE", + "REASON_LOSS_INCONSISTENT", + "REASON_METRICS_MALFORMED", + "REASON_SCHEMA_VERSION", + "REASON_STEP0_ANOMALY", + "REASON_WALLCLOCK_BUDGET", + "STEP0_ANOMALY_FRACTION", + "WALL_CLOCK_BUDGET_MULTIPLIER", + "PlausibilityError", + "check_manifest_plausibility", +] diff --git a/tests/test_prism_plausibility.py b/tests/test_prism_plausibility.py new file mode 100644 index 0000000..e91e12b --- /dev/null +++ b/tests/test_prism_plausibility.py @@ -0,0 +1,435 @@ +"""Verify-only plausibility gate for worker-reported results (architecture.md 3.5). + +Covers VAL-PRISM-009 (implausible manifests are rejected with a distinguishable, plausibility- +specific reason and are never scored) and VAL-PRISM-010 (a well-formed manifest passes the gate +unchanged and finalizes to the exact same score as the legacy finalization path). Offline, no GPU: +the ingestion integration drives the CPU re-exec seam the other coordination tests use and proofs +are signed with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import copy +import io +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.ingestion import ingest_work_unit_result +from prism_challenge.models import SubmissionCreate +from prism_challenge.plausibility import ( + MANIFEST_SCHEMA_VERSION, + REASON_LOSS_AT_BASELINE, + REASON_LOSS_INCONSISTENT, + REASON_METRICS_MALFORMED, + REASON_SCHEMA_VERSION, + REASON_STEP0_ANOMALY, + REASON_WALLCLOCK_BUDGET, + PlausibilityError, + check_manifest_plausibility, +) +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) + +WORKER_KEY = "//WorkerPlausible" +VOCAB = 50257 +BASELINE_NATS = math.log(VOCAB) +BUDGET_SECONDS = 1800.0 + + +# --- manifest fixtures --------------------------------------------------------------------------- + + +def _plausible_manifest() -> dict[str, Any]: + """A well-formed v2 manifest: decreasing loss starting near baseline, wall-clock in budget.""" + + online_loss = [10.4, 8.1, 6.0, 4.2, 3.1, 2.6] + return { + "schema_version": MANIFEST_SCHEMA_VERSION, + "run": {"seed": 1234, "forced_init": True, "stopped_reason": "token_budget"}, + "compute": {"schema": "prism_compute.v1", "wall_clock_seconds": 600.0}, + "metrics": { + "online_loss": online_loss, + "step0_loss": online_loss[0], + "random_init_baseline_nats": BASELINE_NATS, + "prequential_bpb": 1.23, + "bits_per_byte": 1.23, + "covered_bytes": 4096, + }, + "score": { + "schema": "prism_score.v2", + "prequential_bpb": 1.23, + "bits_per_byte": 1.23, + "final_score": 1.0 / (1.0 + 1.23), + }, + "anti_cheat": {"step0_anomaly": False, "no_learning": False, "zero_forward": False}, + } + + +def test_plausible_manifest_passes() -> None: + # No raise for a well-formed manifest. + check_manifest_plausibility(_plausible_manifest(), wall_clock_budget_seconds=BUDGET_SECONDS) + + +def test_plausible_manifest_is_not_mutated() -> None: + manifest = _plausible_manifest() + before = copy.deepcopy(manifest) + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert manifest == before + + +def test_minimal_metrics_only_manifest_passes() -> None: + # A manifest with the schema + a metrics dict but no trajectory/compute fields cannot be shown + # implausible, so it must pass (keeps the existing ingestion fixtures accepted). + minimal = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "metrics": {"prequential_bpb": 1.23}, + } + check_manifest_plausibility(minimal, wall_clock_budget_seconds=BUDGET_SECONDS) + + +def test_rejects_bad_schema_version() -> None: + manifest = _plausible_manifest() + manifest["schema_version"] = "prism_run_manifest.v1" + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_SCHEMA_VERSION + + +def test_rejects_missing_or_malformed_metrics() -> None: + for bad in (None, [], "metrics", 3): + manifest = _plausible_manifest() + if bad is None: + manifest.pop("metrics") + else: + manifest["metrics"] = bad + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_METRICS_MALFORMED + + +def test_rejects_final_loss_at_random_baseline() -> None: + manifest = _plausible_manifest() + # A trajectory that never leaves the from-scratch baseline (~ln(vocab)): no learning happened. + manifest["metrics"]["online_loss"] = [BASELINE_NATS, BASELINE_NATS, BASELINE_NATS] + manifest["metrics"]["step0_loss"] = BASELINE_NATS + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_LOSS_AT_BASELINE + + +def test_rejects_final_loss_above_random_baseline() -> None: + manifest = _plausible_manifest() + manifest["metrics"]["online_loss"] = [10.0, 11.0, 12.0] + manifest["metrics"]["step0_loss"] = 10.0 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_LOSS_AT_BASELINE + + +def test_rejects_no_learning_anti_cheat_flag() -> None: + manifest = _plausible_manifest() + manifest["anti_cheat"]["no_learning"] = True + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_LOSS_AT_BASELINE + + +def test_rejects_step0_anomaly_by_value() -> None: + manifest = _plausible_manifest() + # An initial loss impossibly far below the from-scratch baseline (smuggled-weights signal). + manifest["metrics"]["online_loss"] = [0.9, 0.6, 0.4] + manifest["metrics"]["step0_loss"] = 0.9 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_STEP0_ANOMALY + + +def test_rejects_step0_anomaly_flag() -> None: + manifest = _plausible_manifest() + manifest["anti_cheat"]["step0_anomaly"] = True + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_STEP0_ANOMALY + + +def test_rejects_wallclock_grossly_over_budget() -> None: + manifest = _plausible_manifest() + manifest["compute"]["wall_clock_seconds"] = BUDGET_SECONDS * 50 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_WALLCLOCK_BUDGET + + +def test_rejects_negative_wallclock() -> None: + manifest = _plausible_manifest() + manifest["compute"]["wall_clock_seconds"] = -1.0 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_WALLCLOCK_BUDGET + + +def test_wallclock_check_skipped_without_budget() -> None: + manifest = _plausible_manifest() + manifest["compute"]["wall_clock_seconds"] = BUDGET_SECONDS * 50 + # With no budget known, an over-budget wall-clock cannot be judged implausible. + check_manifest_plausibility(manifest, wall_clock_budget_seconds=None) + + +def test_rejects_log_score_inconsistency() -> None: + manifest = _plausible_manifest() + # The score block's bpb disagrees with the logged metrics bpb: fabricated/self-inconsistent. + manifest["score"]["prequential_bpb"] = 0.01 + manifest["score"]["bits_per_byte"] = 0.01 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_LOSS_INCONSISTENT + + +def test_rejects_step0_vs_online_loss_inconsistency() -> None: + manifest = _plausible_manifest() + manifest["metrics"]["step0_loss"] = manifest["metrics"]["online_loss"][0] + 3.0 + with pytest.raises(PlausibilityError) as exc: + check_manifest_plausibility(manifest, wall_clock_budget_seconds=BUDGET_SECONDS) + assert exc.value.reason == REASON_LOSS_INCONSISTENT + + +# --- ingestion integration (VAL-PRISM-009 / VAL-PRISM-010) --------------------------------------- + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=worker_plane, + ) + + +def _proof_dict(signer, unit_id: str, manifest: dict[str, Any]) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + return proof.model_dump(mode="json") + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any]: + return { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + MANIFEST_PAYLOAD_KEY: manifest, + } + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _score(db_path: Path, submission_id: str): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return row[0] if row else None + + +async def test_ingestion_rejects_implausible_manifests_without_scoring(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + def _bad_schema() -> dict[str, Any]: + m = _plausible_manifest() + m["schema_version"] = "prism_run_manifest.v1" + return m + + def _no_metrics() -> dict[str, Any]: + m = _plausible_manifest() + m.pop("metrics") + return m + + def _at_baseline() -> dict[str, Any]: + m = _plausible_manifest() + m["metrics"]["online_loss"] = [BASELINE_NATS, BASELINE_NATS, BASELINE_NATS] + m["metrics"]["step0_loss"] = BASELINE_NATS + return m + + def _step0() -> dict[str, Any]: + m = _plausible_manifest() + m["metrics"]["online_loss"] = [0.9, 0.6, 0.4] + m["metrics"]["step0_loss"] = 0.9 + return m + + def _wallclock() -> dict[str, Any]: + m = _plausible_manifest() + m["compute"]["wall_clock_seconds"] = BUDGET_SECONDS * 50 + return m + + cases = { + REASON_SCHEMA_VERSION: _bad_schema, + REASON_METRICS_MALFORMED: _no_metrics, + REASON_LOSS_AT_BASELINE: _at_baseline, + REASON_STEP0_ANOMALY: _step0, + REASON_WALLCLOCK_BUDGET: _wallclock, + } + + for expected_reason, build in cases.items(): + submission_id = await _seed(app) + manifest = build() + proof = _proof_dict(signer, submission_id, manifest) + with pytest.raises(PlausibilityError) as exc: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert exc.value.reason == expected_reason + # Never scored; the unit stays eligible for retry (still pending). + assert _score(db_path, submission_id) is None + assert await app.state.repository.submission_status(submission_id) == "pending" + + +async def test_ingestion_accepts_plausible_manifest_identically_to_legacy(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + # Path A: finalize a plausible worker result through the plausibility-gated ingestion path. + gated_id = await _seed(app) + manifest = _plausible_manifest() + before = copy.deepcopy(manifest) + proof = _proof_dict(signer, gated_id, manifest) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=gated_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert outcome.status == "accepted" + assert outcome.finalized is True + assert await app.state.repository.submission_status(gated_id) == "completed" + gated_score = _score(db_path, gated_id) + assert gated_score is not None + # The forwarded manifest / score inputs are passed through UNCHANGED. + assert manifest == before + + # Path B: finalize an identical submission through the legacy in-process path directly. + legacy_id = await _seed(app) + await app.state.worker.process_submission(legacy_id) + legacy_score = _score(db_path, legacy_id) + assert legacy_score is not None + + assert gated_score == pytest.approx(legacy_score) From edc636892fb5dd16f997d0b8e20eb3e2278411dd Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:59:48 +0000 Subject: [PATCH 04/14] feat(prism): probabilistic audit scheduler + mismatch invalidation (M3 prism-proof) Sample finalized worker results at per-tier rates (0.10/0.05/0.02, seeded); sampled results create DISTINCT-id validator audit units on the existing validator_dispatch path (pending-only listing). Audit mismatch invalidates the submission score in its epoch leaderboard and propagates to crown/weights; audit failure/timeout re-audits within bounds and never silently accepts. R=1-degraded results stay audit-eligible at their effective-tier rate; worker hotkeys still cannot publish checkpoints while resume refs reach reassigned units. Fulfills VAL-PRISM-011,012,013,023,024,025,026. --- src/prism_challenge/app.py | 63 +- src/prism_challenge/audit.py | 186 +++++ src/prism_challenge/coordination.py | 26 + src/prism_challenge/db.py | 24 + src/prism_challenge/ingestion.py | 21 + src/prism_challenge/repository.py | 189 +++++ tests/test_prism_audit_scheduler.py | 768 ++++++++++++++++++ ...st_prism_checkpoint_intake_worker_plane.py | 169 ++++ 8 files changed, 1442 insertions(+), 4 deletions(-) create mode 100644 tests/test_prism_audit_scheduler.py create mode 100644 tests/test_prism_checkpoint_intake_worker_plane.py diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index 0732c0b..dbc2a61 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -3,15 +3,19 @@ import base64 import binascii import json -from collections.abc import Callable, Coroutine +from collections.abc import Callable, Coroutine, Mapping from typing import Annotated, Any from fastapi import Depends, FastAPI, Header, HTTPException, Request, status -from .audit import audit_sampler_from_config +from .audit import audit_sampler_from_config, resolve_audit_unit from .auth import authenticate_internal, authenticate_validator from .config import PrismSettings, settings -from .coordination import list_pending_prism_work_units, work_unit_to_payload +from .coordination import ( + audit_work_unit_to_payload, + list_pending_prism_work_units, + work_unit_to_payload, +) from .db import Database from .evaluator.checkpoint_intake import CheckpointIntakeError, CheckpointIntakeService from .evaluator.checkpoint_publisher import ( @@ -152,13 +156,64 @@ async def work_units() -> dict[str, object]: The master coordination plane reads this to create exactly one assignable work unit per submission and assign it - with concurrency 1 - to a single online gpu validator. This endpoint is execution-free: enumerating work units never invokes the broker/executor. + + With the worker plane ON it ALSO exposes pending validator AUDIT units (distinct ids, + validator executor kind) sampled from finalized worker results so they run on the existing + validator_dispatch path (VAL-PRISM-012); resolved/exhausted audits are not listed + (pending-only listing semantics). """ units = await list_pending_prism_work_units(repository) + work_unit_payloads: list[Mapping[str, object]] = [ + work_unit_to_payload(unit) for unit in units + ] + if app_settings.worker_plane.enabled: + for row in await repository.list_pending_audit_units(): + work_unit_payloads.append(audit_work_unit_to_payload(row)) return { "challenge_slug": app_settings.slug, - "work_units": [work_unit_to_payload(unit) for unit in units], + "work_units": work_unit_payloads, } + @app.post( + "/internal/v1/audit_units/{audit_unit_id}/result", + dependencies=[Depends(authenticate_internal)], + ) + async def audit_unit_result(audit_unit_id: str, request: Request) -> dict[str, object]: + """Resolve a validator audit replay for a sampled result (architecture.md 3.5). + + Body: ``{manifest_sha256?: str, success?: bool, error?: str}``. The validator replay is + authoritative: a hash EQUAL to the audited worker hash passes (score untouched); a DIFFERENT + hash invalidates the audited submission's score and propagates to crown/weights + (VAL-PRISM-013/023); a replay failure/timeout (``success: false`` or no ``manifest_sha256``) + NEVER confirms the result -- it is re-audited within bounds, then reaches a terminal + ``failed`` audit state with the submission left unresolved (VAL-PRISM-024). Disabled with + the worker plane off (404). + """ + if not app_settings.worker_plane.enabled: + raise HTTPException(status.HTTP_404_NOT_FOUND, "worker plane disabled") + try: + payload = json.loads(await request.body()) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "invalid JSON result body") from exc + if not isinstance(payload, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "result body must be an object") + replay_hash = payload.get("manifest_sha256") + if replay_hash is not None and not isinstance(replay_hash, str): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "manifest_sha256 must be a string") + failed = payload.get("success") is False or bool(payload.get("failed")) + error = payload.get("error") if isinstance(payload.get("error"), str) else None + try: + resolution = await resolve_audit_unit( + repository, + audit_unit_id=audit_unit_id, + replay_manifest_sha256=replay_hash, + failed=failed, + error=error, + ) + except KeyError as exc: + raise HTTPException(status.HTTP_404_NOT_FOUND, "audit unit not found") from exc + return resolution.to_response() + @app.post( "/internal/v1/work_units/result", dependencies=[Depends(authenticate_internal)] ) diff --git a/src/prism_challenge/audit.py b/src/prism_challenge/audit.py index 8f7bc51..bed29cd 100644 --- a/src/prism_challenge/audit.py +++ b/src/prism_challenge/audit.py @@ -20,6 +20,7 @@ import hashlib from dataclasses import dataclass +from typing import Protocol from .config import WorkerPlaneConfig from .proof import ExecutionProof, has_attestation @@ -27,6 +28,35 @@ #: The three proof tiers the audit rate is keyed on (architecture.md 3.4). TIER_0, TIER_1, TIER_2 = 0, 1, 2 +#: Prefix that makes an audit work-unit id DISTINCT from the primary unit id (== submission_id), +#: which is reserved for the submission's own evaluation unit (VAL-PRISM-012). +AUDIT_UNIT_PREFIX = "audit:" + +#: Audit-unit lifecycle (architecture.md 3.5). ``pending`` units are the only ones exposed on the +#: coordination plane (pending-only listing semantics); the rest are terminal EXCEPT ``pending`` set +#: again on a bounded re-audit after an inconclusive failure/timeout (VAL-PRISM-024). +AUDIT_STATUS_PENDING = "pending" +AUDIT_STATUS_PASSED = "passed" +AUDIT_STATUS_MISMATCH = "mismatch" +AUDIT_STATUS_FAILED = "failed" + +#: Audit resolutions recorded against the unit (distinct from the lifecycle status). +AUDIT_RESOLUTION_PASS = "pass" +AUDIT_RESOLUTION_MISMATCH = "mismatch" +AUDIT_RESOLUTION_INCONCLUSIVE = "inconclusive" + + +def audit_unit_id_for(submission_id: str) -> str: + """Return the audit work-unit id for ``submission_id`` (distinct from the primary unit id).""" + + return f"{AUDIT_UNIT_PREFIX}{submission_id}" + + +def is_audit_unit_id(work_unit_id: str) -> bool: + """Whether ``work_unit_id`` names an audit unit rather than a primary evaluation unit.""" + + return work_unit_id.startswith(AUDIT_UNIT_PREFIX) + def effective_tier(proof: ExecutionProof, *, pinned_image_digest: str | None = None) -> int: """Return the VERIFIED tier for ``proof`` (never higher than what the backing metadata proves). @@ -136,13 +166,169 @@ def audit_sampler_from_config(worker_plane: WorkerPlaneConfig, *, seed: int = 0) ) +class SupportsAuditResolution(Protocol): + """The slice of :class:`~prism_challenge.repository.PrismRepository` audit resolution needs.""" + + async def get_audit_unit(self, audit_unit_id: str) -> dict[str, object] | None: ... + + async def record_audit_resolution( + self, + *, + audit_unit_id: str, + status: str, + attempts: int, + resolution: str | None, + resolved_manifest_sha256: str | None, + error: str | None, + ) -> None: ... + + async def invalidate_submission_score(self, submission_id: str, *, reason: str) -> bool: ... + + +@dataclass(frozen=True) +class AuditResolution: + """The observable outcome of resolving one audit unit against a validator replay.""" + + audit_unit_id: str + submission_id: str + status: str + resolution: str | None + attempts: int + invalidated: bool + terminal: bool + + def to_response(self) -> dict[str, object]: + return { + "audit_unit_id": self.audit_unit_id, + "submission_id": self.submission_id, + "status": self.status, + "resolution": self.resolution, + "attempts": self.attempts, + "invalidated": self.invalidated, + "terminal": self.terminal, + } + + +async def resolve_audit_unit( + repository: SupportsAuditResolution, + *, + audit_unit_id: str, + replay_manifest_sha256: str | None = None, + failed: bool = False, + error: str | None = None, +) -> AuditResolution: + """Resolve an audit unit from a validator's authoritative replay (architecture.md 3.5). + + The validator replay is authoritative. Outcomes: + + * an authoritative replay hash EQUAL to the audited worker hash -> ``passed`` (the audited score + is left untouched); + * an authoritative replay hash DIFFERENT from it -> ``mismatch``: the audited submission's score + is invalidated (VAL-PRISM-013) and the crown/weights recomputed (VAL-PRISM-023); + * a replay FAILURE or TIMEOUT (``failed`` / no ``replay_manifest_sha256``) NEVER confirms the + audited result: the unit is re-audited (back to ``pending``) until ``max_attempts`` is + exhausted, then reaches the terminal, observable ``failed`` state -- the audited submission is + left unresolved (never silently reverted to accepted) and NO fault is attributed, because a + fault requires an authoritative divergent manifest (VAL-PRISM-024). + + Resolving an already-terminal unit is an idempotent no-op. + """ + + unit = await repository.get_audit_unit(audit_unit_id) + if unit is None: + raise KeyError(f"audit unit {audit_unit_id!r} not found") + + submission_id = str(unit["submission_id"]) + status = str(unit["status"]) + attempts = int(unit["attempts"]) # type: ignore[call-overload] + if status in (AUDIT_STATUS_PASSED, AUDIT_STATUS_MISMATCH, AUDIT_STATUS_FAILED): + return AuditResolution( + audit_unit_id=audit_unit_id, + submission_id=submission_id, + status=status, + resolution=(str(unit["resolution"]) if unit["resolution"] is not None else None), + attempts=attempts, + invalidated=status == AUDIT_STATUS_MISMATCH, + terminal=True, + ) + + max_attempts = int(unit["max_attempts"]) # type: ignore[call-overload] + attempts += 1 + inconclusive = failed or not replay_manifest_sha256 + + if inconclusive: + exhausted = attempts >= max_attempts + new_status = AUDIT_STATUS_FAILED if exhausted else AUDIT_STATUS_PENDING + await repository.record_audit_resolution( + audit_unit_id=audit_unit_id, + status=new_status, + attempts=attempts, + resolution=AUDIT_RESOLUTION_INCONCLUSIVE if exhausted else None, + resolved_manifest_sha256=None, + error=error or "audit replay failed or timed out", + ) + return AuditResolution( + audit_unit_id=audit_unit_id, + submission_id=submission_id, + status=new_status, + resolution=AUDIT_RESOLUTION_INCONCLUSIVE if exhausted else None, + attempts=attempts, + invalidated=False, + terminal=exhausted, + ) + + audited_hash = str(unit["audited_manifest_sha256"]) + matches = replay_manifest_sha256 == audited_hash + invalidated = False + if matches: + new_status = AUDIT_STATUS_PASSED + resolution = AUDIT_RESOLUTION_PASS + else: + new_status = AUDIT_STATUS_MISMATCH + resolution = AUDIT_RESOLUTION_MISMATCH + invalidated = await repository.invalidate_submission_score( + submission_id, + reason=f"audit invalidated: manifest mismatch (audit_unit={audit_unit_id})", + ) + await repository.record_audit_resolution( + audit_unit_id=audit_unit_id, + status=new_status, + attempts=attempts, + resolution=resolution, + resolved_manifest_sha256=replay_manifest_sha256, + error=None, + ) + return AuditResolution( + audit_unit_id=audit_unit_id, + submission_id=submission_id, + status=new_status, + resolution=resolution, + attempts=attempts, + invalidated=invalidated, + terminal=True, + ) + + __all__ = [ + "AUDIT_RESOLUTION_INCONCLUSIVE", + "AUDIT_RESOLUTION_MISMATCH", + "AUDIT_RESOLUTION_PASS", + "AUDIT_STATUS_FAILED", + "AUDIT_STATUS_MISMATCH", + "AUDIT_STATUS_PASSED", + "AUDIT_STATUS_PENDING", + "AUDIT_UNIT_PREFIX", "TIER_0", "TIER_1", "TIER_2", "AuditDecision", + "AuditResolution", "AuditSampler", + "SupportsAuditResolution", "audit_sampler_from_config", + "audit_unit_id_for", "effective_tier", + "is_audit_unit_id", "is_tier_downgraded", + "resolve_audit_unit", ] diff --git a/src/prism_challenge/coordination.py b/src/prism_challenge/coordination.py index 9aca1fe..5a7585d 100644 --- a/src/prism_challenge/coordination.py +++ b/src/prism_challenge/coordination.py @@ -138,3 +138,29 @@ def work_unit_to_payload(unit: PrismWorkUnit) -> Mapping[str, Any]: "required_capability": unit.required_capability, "payload": dict(unit.payload), } + + +#: Executor kind an audit unit is dispatched to: audits ALWAYS run on the authoritative validator +#: replay path (``validator_dispatch``), never on a worker (architecture.md 3.5; VAL-PRISM-012). +AUDIT_EXECUTOR_KIND = "validator" + + +def audit_work_unit_to_payload(row: Mapping[str, Any]) -> Mapping[str, Any]: + """Serialise a pending audit unit (an ``audit_units`` row joined with the submission hotkey). + + The audit unit carries a work_unit_id DISTINCT from the audited submission's primary unit id + and is marked for validator execution (``required_capability == "gpu"``, executor kind + ``validator``) referencing the audited submission id (VAL-PRISM-012). + """ + + submission_id = str(row["submission_id"]) + return { + "work_unit_id": str(row["audit_unit_id"]), + "submission_id": submission_id, + "submission_ref": str(row.get("hotkey") or ""), + "required_capability": str(row.get("required_capability") or PRISM_WORK_UNIT_CAPABILITY), + "executor_kind": AUDIT_EXECUTOR_KIND, + "audit": True, + "audited_submission_id": submission_id, + "payload": {"audit": True, "audited_submission_id": submission_id}, + } diff --git a/src/prism_challenge/db.py b/src/prism_challenge/db.py index 35919f7..de97634 100644 --- a/src/prism_challenge/db.py +++ b/src/prism_challenge/db.py @@ -184,6 +184,17 @@ "tier_downgraded INTEGER NOT NULL DEFAULT 0, worker_pubkey TEXT, accepted_at TEXT NOT NULL);" "CREATE INDEX IF NOT EXISTS idx_work_unit_results_submission " "ON work_unit_results(submission_id);" + "CREATE TABLE IF NOT EXISTS audit_units (" + "audit_unit_id TEXT PRIMARY KEY, submission_id TEXT NOT NULL," + "origin_work_unit_id TEXT NOT NULL, epoch_id INTEGER NOT NULL," + "audited_manifest_sha256 TEXT NOT NULL, effective_tier INTEGER NOT NULL," + "replication INTEGER NOT NULL DEFAULT 2, required_capability TEXT NOT NULL DEFAULT 'gpu'," + "executor_kind TEXT NOT NULL DEFAULT 'validator', status TEXT NOT NULL," + "attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL DEFAULT 3," + "resolved_manifest_sha256 TEXT, resolution TEXT, error TEXT," + "created_at TEXT NOT NULL, updated_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" + "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" ) @@ -320,6 +331,19 @@ async def _run_migrations(conn: aiosqlite.Connection) -> None: "CREATE INDEX IF NOT EXISTS idx_work_unit_results_submission " "ON work_unit_results(submission_id);" ) + await conn.executescript( + "CREATE TABLE IF NOT EXISTS audit_units (" + "audit_unit_id TEXT PRIMARY KEY, submission_id TEXT NOT NULL," + "origin_work_unit_id TEXT NOT NULL, epoch_id INTEGER NOT NULL," + "audited_manifest_sha256 TEXT NOT NULL, effective_tier INTEGER NOT NULL," + "replication INTEGER NOT NULL DEFAULT 2, required_capability TEXT NOT NULL DEFAULT 'gpu'," + "executor_kind TEXT NOT NULL DEFAULT 'validator', status TEXT NOT NULL," + "attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL DEFAULT 3," + "resolved_manifest_sha256 TEXT, resolution TEXT, error TEXT," + "created_at TEXT NOT NULL, updated_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" + "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" + ) await conn.executescript( "CREATE TABLE IF NOT EXISTS gpu_leases (" "id TEXT PRIMARY KEY, submission_id TEXT NOT NULL, job_id TEXT, target_id TEXT," diff --git a/src/prism_challenge/ingestion.py b/src/prism_challenge/ingestion.py index 7a9df74..cd6bf3b 100644 --- a/src/prism_challenge/ingestion.py +++ b/src/prism_challenge/ingestion.py @@ -103,6 +103,7 @@ class IngestionOutcome: finalized: bool submission_status: str | None = None audit_sampled: bool | None = None + audit_unit_id: str | None = None reason: str | None = None def to_response(self) -> dict[str, Any]: @@ -119,6 +120,8 @@ def to_response(self) -> dict[str, Any]: } if self.audit_sampled is not None: payload["audit_sampled"] = self.audit_sampled + if self.audit_unit_id is not None: + payload["audit_unit_id"] = self.audit_unit_id if self.reason is not None: payload["reason"] = self.reason return payload @@ -220,6 +223,10 @@ async def ingest_work_unit_result( claimed_tier = int(proof.tier) downgraded = claimed_tier != tier submission_id = work_unit_id + # Replication at acceptance (R=1 degraded or R=2 reconciled), forwarded by base for + # observability. It never affects audit eligibility: R=1 results are audited at their + # effective-tier rate exactly like R=2 ones (VAL-PRISM-026). + replication = _as_int(result.get("replication"), 2) repository = worker.repository existing = await repository.get_work_unit_result(work_unit_id) @@ -283,10 +290,23 @@ async def ingest_work_unit_result( worker_pubkey=proof.worker_signature.worker_pubkey, ) audit_sampled: bool | None = None + audit_unit_id: str | None = None if audit_sampler is not None: audit_sampled = audit_sampler.should_sample( work_unit_id=work_unit_id, effective_tier=tier ) + # A sampled accepted result gets a validator audit unit on the existing dispatch path with a + # DISTINCT id; the audited submission is NOT reverted to pending (VAL-PRISM-012). R=1 + # (replication-degraded) results are sampled and audited at their effective-tier rate just + # like R=2-reconciled ones -- they are never exempted (VAL-PRISM-026). + if audit_sampled: + audit_unit_id = await repository.create_audit_unit( + submission_id=submission_id, + origin_work_unit_id=work_unit_id, + audited_manifest_sha256=proof.manifest_sha256, + effective_tier=tier, + replication=replication, + ) return IngestionOutcome( status="accepted", work_unit_id=work_unit_id, @@ -298,6 +318,7 @@ async def ingest_work_unit_result( finalized=result_id is not None, submission_status=submission_status, audit_sampled=audit_sampled, + audit_unit_id=audit_unit_id, ) diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index c04e52c..d400d75 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -1085,6 +1085,195 @@ async def record_work_unit_result( ), ) + async def create_audit_unit( + self, + *, + submission_id: str, + origin_work_unit_id: str, + audited_manifest_sha256: str, + effective_tier: int, + replication: int = 2, + max_attempts: int = 3, + ) -> str: + """Create the validator audit unit for a sampled result (idempotent per submission). + + The audit unit id is DISTINCT from the primary unit id (``== submission_id``), so an audit + never collides with the submission's own evaluation unit (VAL-PRISM-012). ``INSERT OR + IGNORE`` keeps a single audit unit per sampled submission; the audited submission is NOT + reverted to pending by this call. Returns the audit unit id. + """ + from .audit import AUDIT_STATUS_PENDING, audit_unit_id_for + + audit_unit_id = audit_unit_id_for(submission_id) + now = now_iso() + async with self.database.connect() as conn: + epoch_rows = await conn.execute_fetchall( + "SELECT epoch_id FROM submissions WHERE id=?", (submission_id,) + ) + epoch_list = list(epoch_rows) + epoch_id = int(cast(SupportsInt, epoch_list[0]["epoch_id"])) if epoch_list else 0 + await conn.execute( + "INSERT OR IGNORE INTO audit_units(" + "audit_unit_id, submission_id, origin_work_unit_id, epoch_id," + "audited_manifest_sha256, effective_tier, replication, required_capability," + "executor_kind, status, attempts, max_attempts, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, 'gpu', 'validator', ?, 0, ?, ?, ?)", + ( + audit_unit_id, + submission_id, + origin_work_unit_id, + epoch_id, + audited_manifest_sha256, + int(effective_tier), + int(replication), + AUDIT_STATUS_PENDING, + int(max_attempts), + now, + now, + ), + ) + return audit_unit_id + + async def get_audit_unit(self, audit_unit_id: str) -> dict[str, object] | None: + """Return one audit unit row (``None`` when it does not exist).""" + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT * FROM audit_units WHERE audit_unit_id=?", (audit_unit_id,) + ) + row_list = list(rows) + return dict(cast(Any, row_list[0])) if row_list else None + + async def list_pending_audit_units(self) -> list[dict[str, object]]: + """Return pending audit units joined with the audited submission's hotkey (oldest first). + + Only ``pending`` audit units are exposed on the coordination plane; a resolved (or + exhausted) audit is no longer listed (pending-only listing semantics; VAL-PRISM-012). + """ + from .audit import AUDIT_STATUS_PENDING + + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT au.*, s.hotkey AS hotkey FROM audit_units au " + "JOIN submissions s ON s.id=au.submission_id " + "WHERE au.status=? ORDER BY au.created_at, au.audit_unit_id", + (AUDIT_STATUS_PENDING,), + ) + return [dict(cast(Any, row)) for row in rows] + + async def record_audit_resolution( + self, + *, + audit_unit_id: str, + status: str, + attempts: int, + resolution: str | None, + resolved_manifest_sha256: str | None, + error: str | None, + ) -> None: + """Persist an audit unit's new lifecycle state after a resolution attempt.""" + async with self.database.connect() as conn: + await conn.execute( + "UPDATE audit_units SET status=?, attempts=?, resolution=?, " + "resolved_manifest_sha256=?, error=?, updated_at=? WHERE audit_unit_id=?", + ( + status, + int(attempts), + resolution, + resolved_manifest_sha256, + error, + now_iso(), + audit_unit_id, + ), + ) + + async def invalidate_submission_score(self, submission_id: str, *, reason: str) -> bool: + """Invalidate a finalized submission's score and recompute crown/weights aggregates. + + Deletes the ``scores`` row and moves the submission to ``failed`` (so it drops out of the + epoch leaderboard, ``score_rows`` and weights; VAL-PRISM-013), then recomputes the affected + architecture family's ``q_arch_best``/``canonical_submission_id`` and its training variants + from the REMAINING valid (completed + scored) submissions so the crown falls back to the + best non-invalidated submission or to the BURN state, and ``is_current_best`` moves off the + invalidated submission (VAL-PRISM-023). Returns ``True`` when a submission row existed. + """ + now = now_iso() + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT arch_hash FROM submissions WHERE id=?", (submission_id,) + ) + row_list = list(rows) + if not row_list: + return False + arch_hash = row_list[0]["arch_hash"] + await conn.execute("DELETE FROM scores WHERE submission_id=?", (submission_id,)) + await conn.execute( + "UPDATE submissions SET status=?, error=?, updated_at=? WHERE id=?", + (SubmissionStatus.FAILED.value, reason, now, submission_id), + ) + if arch_hash: + await self._recompute_family_after_invalidation( + conn, + family_hash=str(arch_hash), + invalidated_submission_id=submission_id, + now=now, + ) + return True + + async def _recompute_family_after_invalidation( + self, + conn: aiosqlite.Connection, + *, + family_hash: str, + invalidated_submission_id: str, + now: str, + ) -> None: + """Recompute an architecture family's crown aggregates after an invalidation.""" + fam_rows = await conn.execute_fetchall( + "SELECT id FROM architecture_families WHERE family_hash=?", (family_hash,) + ) + fam_list = list(fam_rows) + if not fam_list: + return + architecture_id = str(fam_list[0]["id"]) + best_rows = await conn.execute_fetchall( + "SELECT s.id AS sid, sc.final_score AS fs FROM submissions s " + "JOIN scores sc ON sc.submission_id=s.id " + "WHERE s.arch_hash=? AND s.status=? " + "ORDER BY sc.final_score DESC, s.created_at ASC, s.id ASC LIMIT 1", + (family_hash, SubmissionStatus.COMPLETED.value), + ) + best = list(best_rows) + if best: + best_score = float(cast(SupportsFloat, best[0]["fs"])) + await conn.execute( + "UPDATE architecture_families SET canonical_submission_id=?, q_arch_best=?, " + "updated_at=? WHERE id=?", + (str(best[0]["sid"]), best_score, now, architecture_id), + ) + else: + # No valid submission remains for the family: drop q_arch_best to 0 so the crown falls + # to another family or BURNs (get_weights treats a non-positive best as no crown). + await conn.execute( + "UPDATE architecture_families SET q_arch_best=0.0, updated_at=? WHERE id=?", + (now, architecture_id), + ) + # The invalidated submission was a training variant's representative; drop that variant (its + # only evidence is gone) and recompute is_current_best across the family's survivors. + await conn.execute( + "DELETE FROM training_variants WHERE architecture_id=? AND submission_id=?", + (architecture_id, invalidated_submission_id), + ) + await conn.execute( + "UPDATE training_variants SET is_current_best=0 WHERE architecture_id=?", + (architecture_id,), + ) + await conn.execute( + "UPDATE training_variants SET is_current_best=1 WHERE id=(" + "SELECT id FROM training_variants WHERE architecture_id=? " + "ORDER BY q_recipe DESC, created_at ASC, id ASC LIMIT 1)", + (architecture_id,), + ) + async def container_job_attempt_count(self, submission_id: str, level: str) -> int: async with self.database.connect() as conn: rows = await conn.execute_fetchall( diff --git a/tests/test_prism_audit_scheduler.py b/tests/test_prism_audit_scheduler.py new file mode 100644 index 0000000..c6e283a --- /dev/null +++ b/tests/test_prism_audit_scheduler.py @@ -0,0 +1,768 @@ +"""Probabilistic audit scheduler, audit-unit creation, and audit-mismatch invalidation. + +Covers the mission audit assertions (architecture.md 3.4/3.5): + +* VAL-PRISM-011 - the scheduler samples finalized results at the configured per-tier rates, is + reproducible under a deterministic seed, and a 0.0 rate samples exactly nothing. +* VAL-PRISM-012 - a sampled accepted result creates a validator audit unit on the existing dispatch + path with a DISTINCT work_unit_id (authenticated internal listing, pending-only semantics), the + audited submission is NOT reverted to pending, and a non-sampled result yields none. +* VAL-PRISM-013 - an audit manifest MISMATCH invalidates the submission's score in its epoch + leaderboard; a matching audit leaves it untouched. +* VAL-PRISM-023 - invalidation propagates to the architecture crown and weights (q_arch_best / + canonical / training_variants recomputed; a matching control is unchanged). +* VAL-PRISM-024 - an audit failure/timeout never silently accepts: it re-audits within bounds, then + reaches a terminal observable ``failed`` state with the audited submission left unresolved. +* VAL-PRISM-026 - R=1-degraded results remain audit-eligible at their effective-tier rate and a + forced-sample R=1 result creates an audit unit exactly like an R=2 one. + +Offline, no GPU: finalization runs through the CPU re-exec seam the other prism coordination tests +use, and proofs are signed with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import io +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.audit import ( + AUDIT_STATUS_FAILED, + AUDIT_STATUS_MISMATCH, + AUDIT_STATUS_PASSED, + AUDIT_STATUS_PENDING, + AuditSampler, + audit_unit_id_for, + resolve_audit_unit, +) +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) +from prism_challenge.weights import get_weights + +WORKER_KEY = "//WorkerAudit" +EPOCH_SECONDS = 60 + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'audit.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + epoch_seconds=EPOCH_SECONDS, + worker_plane=worker_plane, + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + return { + "schema_version": "prism_run_manifest.v2", + "metrics": {"prequential_bpb": 1.23, "marker": marker}, + "steps": [{"step": 0, "loss": 3.0}, {"step": 1, "loss": 2.0}], + } + + +def _proof_dict(signer, unit_id: str, manifest: dict[str, Any]) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + return proof.model_dump(mode="json") + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any], **extra: Any) -> dict[str, Any]: + return { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + MANIFEST_PAYLOAD_KEY: manifest, + **extra, + } + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _always() -> AuditSampler: + return AuditSampler(audit_rate_tier0=1.0, audit_rate_tier1=1.0, audit_rate_tier2=1.0, seed=0) + + +def _never() -> AuditSampler: + return AuditSampler(audit_rate_tier0=0.0, audit_rate_tier1=0.0, audit_rate_tier2=0.0, seed=0) + + +async def _finalize( + app, + signer, + *, + hotkey: str = "hk-owner", + sampler: AuditSampler | None = None, + replication: int = 2, +): + """Seed a submission and ingest a well-formed worker result (finalized + optionally sampled).""" + from prism_challenge.ingestion import ingest_work_unit_result + + submission_id = await _seed(app, hotkey) + manifest = _manifest() + proof = _proof_dict(signer, submission_id, manifest) + extra: dict[str, Any] = {"replication": replication} + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=hotkey, + result=_result(proof, manifest, **extra), + audit_sampler=sampler, + ) + return submission_id, manifest, outcome + + +def _score(db_path: Path, submission_id: str): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return row[0] if row else None + + +# --- VAL-PRISM-011: sampler follows the configured per-tier rates, seeded ------------------------- + + +def test_sampling_matches_configured_per_tier_rates_seeded() -> None: + sampler = AuditSampler( + audit_rate_tier0=0.10, audit_rate_tier1=0.05, audit_rate_tier2=0.02, seed=2026 + ) + n = 6000 + + def _bound(p: float) -> float: + # 4-sigma binomial interval around the configured rate for this N. + return 4.0 * (p * (1.0 - p) / n) ** 0.5 + + for tier, rate in ((0, 0.10), (1, 0.05), (2, 0.02)): + hits = sum( + sampler.should_sample(work_unit_id=f"t{tier}-{i}", effective_tier=tier) + for i in range(n) + ) + observed = hits / n + assert abs(observed - rate) < _bound(rate), (tier, observed, rate) + + +def test_zero_rate_samples_nothing_and_seed_reproduces() -> None: + zero = _never() + assert not any( + zero.should_sample(work_unit_id=f"u{i}", effective_tier=t) + for i in range(2000) + for t in (0, 1, 2) + ) + a = AuditSampler(audit_rate_tier0=0.10, seed=11) + b = AuditSampler(audit_rate_tier0=0.10, seed=11) + c = AuditSampler(audit_rate_tier0=0.10, seed=12) + ids = [f"unit-{i}" for i in range(400)] + sa = [a.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + sb = [b.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + sc = [c.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + assert sa == sb # same seed => identical sample set + assert sa != sc # a different seed shifts it + + +# --- VAL-PRISM-012: sampled result creates a distinct validator audit unit on the dispatch path --- + + +async def test_sampled_result_creates_distinct_validator_audit_unit(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + submission_id, _, outcome = await _finalize(app, signer, sampler=_always()) + assert outcome.audit_sampled is True + assert outcome.audit_unit_id == audit_unit_id_for(submission_id) + # The audit unit id is DISTINCT from the primary unit id (== submission_id). + assert outcome.audit_unit_id != submission_id + # The audited submission is NOT reverted to pending by audit creation. + assert await repository.submission_status(submission_id) == "completed" + + pending = await repository.list_pending_audit_units() + assert len(pending) == 1 + row = pending[0] + assert row["audit_unit_id"] == audit_unit_id_for(submission_id) + assert row["submission_id"] == submission_id + assert row["required_capability"] == "gpu" + assert row["executor_kind"] == "validator" + + +def test_audit_unit_visible_on_internal_work_units_and_absent_when_not_sampled( + tmp_path, monkeypatch +) -> None: + import anyio + from fastapi.testclient import TestClient + + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + signer = worker_signer_from_key(WORKER_KEY) + headers = {"Authorization": "Bearer secret"} + + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + with TestClient(create_app(settings)) as client: + app = client.app + + async def _drive() -> tuple[str, str]: + sampled_id, _, out1 = await _finalize(app, signer, hotkey="hk-a", sampler=_always()) + control_id, _, out2 = await _finalize(app, signer, hotkey="hk-b", sampler=_never()) + assert out1.audit_sampled is True and out2.audit_sampled is False + return sampled_id, control_id + + sampled_id, control_id = anyio.run(_drive) + + # Authenticated internal listing shows exactly the sampled submission's audit unit. + unauthorized = client.get("/internal/v1/work_units") + assert unauthorized.status_code == 401 + listed = client.get("/internal/v1/work_units", headers=headers) + assert listed.status_code == 200, listed.text + audit_units = [ + unit for unit in listed.json()["work_units"] if unit.get("audit") is True + ] + assert len(audit_units) == 1 + unit = audit_units[0] + assert unit["work_unit_id"] == audit_unit_id_for(sampled_id) + assert unit["work_unit_id"] != sampled_id + assert unit["submission_id"] == sampled_id + assert unit["required_capability"] == "gpu" + assert unit["executor_kind"] == "validator" + # The non-sampled control never produced an audit unit. + assert all(u["submission_id"] != control_id for u in audit_units) + + +# --- VAL-PRISM-013: audit mismatch invalidates the submission's epoch-leaderboard score ----------- + + +async def test_audit_mismatch_invalidates_score_in_epoch_leaderboard(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit.sqlite3" + + submission_id, _, outcome = await _finalize(app, signer, sampler=_always()) + epoch_id = (await repository.get_submission(submission_id)).epoch_id + # Before audit: the submission holds a score and ranks in its epoch leaderboard. + assert _score(db_path, submission_id) is not None + board_before = await repository.leaderboard(epoch_id) + assert any(row["id"] == submission_id for row in board_before) + + # A validator replay with a DIFFERENT manifest hash resolves to a mismatch and invalidates. + resolution = await resolve_audit_unit( + repository, + audit_unit_id=outcome.audit_unit_id, + replay_manifest_sha256="f" * 64, + ) + assert resolution.status == AUDIT_STATUS_MISMATCH + assert resolution.invalidated is True + assert _score(db_path, submission_id) is None + assert await repository.submission_status(submission_id) == "failed" + board_after = await repository.leaderboard(epoch_id) + assert all(row["id"] != submission_id for row in board_after) + + +async def test_matching_audit_leaves_score_untouched(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit.sqlite3" + + submission_id, manifest, outcome = await _finalize(app, signer, sampler=_always()) + score_before = _score(db_path, submission_id) + assert score_before is not None + + # A validator replay with the SAME manifest hash passes: the score is untouched. + resolution = await resolve_audit_unit( + repository, + audit_unit_id=outcome.audit_unit_id, + replay_manifest_sha256=compute_manifest_sha256(manifest), + ) + assert resolution.status == AUDIT_STATUS_PASSED + assert resolution.invalidated is False + assert _score(db_path, submission_id) == pytest.approx(score_before) + assert await repository.submission_status(submission_id) == "completed" + + +# --- VAL-PRISM-023: invalidation propagates to the architecture crown + weights ------------------- + + +async def test_invalidation_propagates_to_crown_and_weights(tmp_path) -> None: + from prism_challenge.db import Database + from prism_challenge.repository import PrismRepository + + database = Database(tmp_path / "crown.sqlite3") + await database.init() + repository = PrismRepository(database, epoch_seconds=EPOCH_SECONDS) + + # Family A crowned by hk-alice's submission (0.9); family B held by hk-bob (0.4). + await _seed_family(repository, family="A", owner="hk-alice", submission="sA", score=0.9) + await _seed_family(repository, family="B", owner="hk-bob", submission="sB", score=0.4) + + weights_before = await get_weights(repository, EPOCH_SECONDS) + assert weights_before # hk-alice crowned + best_before = await repository.best_architecture() + assert best_before["owner_hotkey"] == "hk-alice" + + # Audit invalidates hk-alice's sole submission: the crown must fall back to hk-bob (family B). + invalidated = await repository.invalidate_submission_score( + "sA", reason="audit invalidated: manifest mismatch" + ) + assert invalidated is True + + best_after = await repository.best_architecture() + assert best_after["owner_hotkey"] == "hk-bob" + weights_after = await get_weights(repository, EPOCH_SECONDS) + assert "hk-alice" not in weights_after + assert weights_after.get("hk-bob", 0.0) > 0.0 + # The invalidated submission's training variant no longer flags current-best. + variants = await repository.list_training_variants("af-A") + assert all(not v["is_current_best"] for v in variants) + + +async def test_invalidation_burns_when_no_valid_submission_remains(tmp_path) -> None: + from prism_challenge.db import Database + from prism_challenge.repository import PrismRepository + + database = Database(tmp_path / "burn.sqlite3") + await database.init() + repository = PrismRepository(database, epoch_seconds=EPOCH_SECONDS) + + await _seed_family(repository, family="A", owner="hk-alice", submission="sA", score=0.9) + assert await get_weights(repository, EPOCH_SECONDS) # crowned + + await repository.invalidate_submission_score("sA", reason="audit invalidated") + # No valid submission remains anywhere -> BURN (empty weights). + assert await get_weights(repository, EPOCH_SECONDS) == {} + + +async def test_matching_audit_control_leaves_crown_untouched(tmp_path) -> None: + from prism_challenge.db import Database + from prism_challenge.repository import PrismRepository + + database = Database(tmp_path / "control.sqlite3") + await database.init() + repository = PrismRepository(database, epoch_seconds=EPOCH_SECONDS) + await _seed_family(repository, family="A", owner="hk-alice", submission="sA", score=0.9) + await _seed_family(repository, family="B", owner="hk-bob", submission="sB", score=0.4) + + before = await get_weights(repository, EPOCH_SECONDS) + best_before = await repository.best_architecture() + # A matching audit does not call invalidate_submission_score, so nothing changes. + after = await get_weights(repository, EPOCH_SECONDS) + best_after = await repository.best_architecture() + assert before == after + assert best_before["owner_hotkey"] == best_after["owner_hotkey"] == "hk-alice" + + +# --- VAL-PRISM-024: audit failure / timeout never silently accepts -------------------------------- + + +async def test_audit_failure_reaudits_then_terminal_failed_without_accepting( + tmp_path, monkeypatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit.sqlite3" + + submission_id, _, outcome = await _finalize(app, signer, sampler=_always()) + audit_unit_id = outcome.audit_unit_id + unit = await repository.get_audit_unit(audit_unit_id) + max_attempts = int(unit["max_attempts"]) + assert max_attempts >= 2 + score_before = _score(db_path, submission_id) + + # Every failure/timeout short of exhaustion re-audits (back to pending), never accepting. + for attempt in range(1, max_attempts): + res = await resolve_audit_unit(repository, audit_unit_id=audit_unit_id, failed=True) + assert res.invalidated is False + assert res.terminal is False + assert res.status == AUDIT_STATUS_PENDING + assert res.attempts == attempt + # Still eligible for re-audit: listed as pending, submission unchanged. + assert any( + u["audit_unit_id"] == audit_unit_id + for u in await repository.list_pending_audit_units() + ) + + # On exhaustion the audit is terminally failed; the audited submission is NEVER accepted-by- + # -failure: its score/status are unchanged (unresolved), not confirmed and not invalidated. + final = await resolve_audit_unit( + repository, audit_unit_id=audit_unit_id, replay_manifest_sha256=None, failed=True + ) + assert final.status == AUDIT_STATUS_FAILED + assert final.terminal is True + assert final.invalidated is False + assert _score(db_path, submission_id) == pytest.approx(score_before) + assert await repository.submission_status(submission_id) == "completed" + # A terminal audit is no longer listed as pending (no unbounded retry loop). + assert all( + u["audit_unit_id"] != audit_unit_id + for u in await repository.list_pending_audit_units() + ) + # Resolving a terminal unit again is an idempotent no-op. + again = await resolve_audit_unit(repository, audit_unit_id=audit_unit_id, failed=True) + assert again.status == AUDIT_STATUS_FAILED + assert again.terminal is True + + +# --- VAL-PRISM-026: R=1-degraded results stay audit-eligible at their effective-tier rate --------- + + +def test_r1_and_r2_results_sampled_at_same_effective_tier_rate() -> None: + sampler = AuditSampler(audit_rate_tier0=0.10, seed=4242) + n = 6000 + + def _bound(p: float) -> float: + return 4.0 * (p * (1.0 - p) / n) ** 0.5 + + # A single population keyed by unit id; R is just a label the base plane attaches, so the + # scheduler samples R=1 and R=2 identically at the effective tier's rate (never exempting R=1). + r1_hits = sum( + sampler.should_sample(work_unit_id=f"r1-{i}", effective_tier=0) for i in range(n) + ) + r2_hits = sum( + sampler.should_sample(work_unit_id=f"r2-{i}", effective_tier=0) for i in range(n) + ) + assert abs(r1_hits / n - 0.10) < _bound(0.10) + assert abs(r2_hits / n - 0.10) < _bound(0.10) + + +async def test_forced_sample_r1_result_creates_audit_unit(tmp_path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + # A replication-degraded (R=1) accepted result is sampled and audited like any other. + submission_id, _, outcome = await _finalize(app, signer, sampler=_always(), replication=1) + assert outcome.audit_sampled is True + row = await repository.get_audit_unit(outcome.audit_unit_id) + assert row is not None + assert int(row["replication"]) == 1 + assert row["status"] == AUDIT_STATUS_PENDING + assert row["executor_kind"] == "validator" + + +# --- Audit resolution HTTP route (curl surface for VAL-PRISM-013/024) ----------------------------- + + +def test_audit_result_route_invalidates_and_drops_from_leaderboard(tmp_path, monkeypatch) -> None: + import anyio + from fastapi.testclient import TestClient + + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + signer = worker_signer_from_key(WORKER_KEY) + headers = {"Authorization": "Bearer secret"} + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + with TestClient(create_app(settings)) as client: + app = client.app + + async def _drive() -> tuple[str, str, int]: + sid, _, out = await _finalize(app, signer, sampler=_always()) + epoch = (await app.state.repository.get_submission(sid)).epoch_id + return sid, out.audit_unit_id, epoch + + submission_id, audit_unit_id, epoch_id = anyio.run(_drive) + + board = client.get(f"/v1/leaderboard?epoch_id={epoch_id}").json() + assert any(e["submission_id"] == submission_id for e in board["entries"]) + + # Unauthenticated resolution is rejected. + assert ( + client.post(f"/internal/v1/audit_units/{audit_unit_id}/result", json={}).status_code + == 401 + ) + # A mismatching validator replay invalidates via the route. + resolved = client.post( + f"/internal/v1/audit_units/{audit_unit_id}/result", + json={"manifest_sha256": "e" * 64}, + headers=headers, + ) + assert resolved.status_code == 200, resolved.text + assert resolved.json()["status"] == AUDIT_STATUS_MISMATCH + assert resolved.json()["invalidated"] is True + + # The submission is gone from its epoch leaderboard, and reads as failed (not completed). + board_after = client.get(f"/v1/leaderboard?epoch_id={epoch_id}").json() + assert all(e["submission_id"] != submission_id for e in board_after["entries"]) + detail = client.get(f"/v1/submissions/{submission_id}").json() + assert detail["status"] == "failed" + assert detail["final_score"] is None + + +def test_audit_result_route_disabled_when_worker_plane_off(tmp_path) -> None: + from fastapi.testclient import TestClient + + settings = _settings(tmp_path, worker_plane=WorkerPlaneConfig(enabled=False)) + with TestClient(create_app(settings)) as client: + resp = client.post( + "/internal/v1/audit_units/audit:sub-x/result", + json={"manifest_sha256": "a" * 64}, + headers={"Authorization": "Bearer secret"}, + ) + assert resp.status_code == 404 + + +def test_work_units_omits_audit_units_when_worker_plane_off(tmp_path) -> None: + import anyio + from fastapi.testclient import TestClient + + settings = _settings(tmp_path, worker_plane=WorkerPlaneConfig(enabled=False)) + headers = {"Authorization": "Bearer secret"} + with TestClient(create_app(settings)) as client: + app = client.app + + async def _seed() -> str: + repo = app.state.repository + sub = await repo.create_submission( + "hk-a", SubmissionCreate(code=_bundle(), filename="a.py") + ) + # An audit unit row exists but must stay invisible while the flag is OFF. + await repo.create_audit_unit( + submission_id=sub.id, + origin_work_unit_id=sub.id, + audited_manifest_sha256="a" * 64, + effective_tier=0, + ) + return sub.id + + submission_id = anyio.run(_seed) + listed = client.get("/internal/v1/work_units", headers=headers).json() + assert all(not unit.get("audit") for unit in listed["work_units"]) + # The primary pending unit is still exposed exactly as legacy. + assert any(unit["submission_id"] == submission_id for unit in listed["work_units"]) + + +# --- helpers -------------------------------------------------------------------------------------- + + +async def _seed_family( + repository: Any, + *, + family: str, + owner: str, + submission: str, + score: float, + epoch_id: int = 1, +) -> None: + """Directly seed a completed submission + score + architecture family + training variant.""" + created = "2026-06-27T00:00:00+00:00" + architecture_id = f"af-{family}" + family_hash = f"fh-{family}" + async with repository.database.connect() as conn: + await conn.execute( + "INSERT OR IGNORE INTO epochs(id, starts_at, ends_at, status) VALUES (?, ?, ?, ?)", + (epoch_id, created, created, "open"), + ) + await conn.execute( + "INSERT OR IGNORE INTO miners(hotkey, first_seen, last_seen) VALUES (?, ?, ?)", + (owner, created, created), + ) + await conn.execute( + "INSERT INTO submissions(" + "id, hotkey, epoch_id, filename, code, code_hash, arch_hash, metadata, status, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + submission, + owner, + epoch_id, + "project.zip", + "x", + submission, + family_hash, + "{}", + "completed", + created, + created, + ), + ) + await conn.execute( + "INSERT INTO scores(" + "submission_id, q_arch, q_recipe, anti_cheat_multiplier, diversity_bonus, " + "penalty, final_score, metrics, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (submission, score, 0.0, 1.0, 0.0, 0.0, score, "{}", created), + ) + await conn.execute( + "INSERT INTO architecture_families(" + "id, family_hash, arch_fingerprint, behavior_fingerprint, owner_hotkey, " + "owner_submission_id, canonical_submission_id, q_arch_best, display_name, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + architecture_id, + family_hash, + f"fp-{family}", + f"bp-{family}", + owner, + submission, + submission, + score, + f"arch-{family}", + created, + created, + ), + ) + await conn.execute( + "INSERT INTO training_variants(" + "id, architecture_id, training_hash, owner_hotkey, submission_id, q_recipe, " + "metric_mean, metric_std, is_current_best, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + f"tv-{family}", + architecture_id, + f"th-{family}", + owner, + submission, + score, + score, + 0.0, + 1, + created, + created, + ), + ) diff --git a/tests/test_prism_checkpoint_intake_worker_plane.py b/tests/test_prism_checkpoint_intake_worker_plane.py new file mode 100644 index 0000000..be7f8e7 --- /dev/null +++ b/tests/test_prism_checkpoint_intake_worker_plane.py @@ -0,0 +1,169 @@ +"""Checkpoint intake trust boundary + resume-ref flow survive the worker plane (VAL-PRISM-025). + +(a) The checkpoint publish intake (`POST /internal/v1/checkpoints`) stays validator-permit gated + even with the worker plane ON: a worker-bound / non-permitted hotkey is rejected 403 and records + no checkpoint ref (worker-plane enablement does not widen the permit set). +(b) A submission that HAS a validator-published checkpoint still exposes `resume_checkpoint_ref` in + its `GET /internal/v1/work_units` payload so a reassigned executor resumes; a fresh unit carries + none. + +Offline: dev (hmac) signatures via ``allow_insecure_signatures`` (no bittensor keys); no GPU. +""" + +from __future__ import annotations + +import base64 +import hmac +import json +import sqlite3 +import time +from hashlib import sha256 +from pathlib import Path + +import anyio +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.auth import canonical_checkpoint_message +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.coordination import RESUME_CHECKPOINT_PAYLOAD_KEY +from prism_challenge.models import SubmissionCreate + +INTERNAL_TOKEN = "secret" +VALIDATOR_HOTKEY = "hk-validator" +WORKER_HOTKEY = "hk-worker" + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'ckpt.sqlite3'}", + shared_token=INTERNAL_TOKEN, + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + validator_hotkeys=[VALIDATOR_HOTKEY], + # Worker plane ON: the trust boundary must NOT widen. + worker_plane=WorkerPlaneConfig(enabled=True), + ) + + +def _bundle() -> str: + code = "def build_model(ctx):\n return None\n" + return base64.b64encode(code.encode()).decode("ascii") + + +def _signed_checkpoint_headers(body: bytes, hotkey: str, nonce: str = "cn1") -> dict[str, str]: + timestamp = str(int(time.time())) + message = canonical_checkpoint_message( + hotkey=hotkey, nonce=nonce, timestamp=timestamp, body=body + ) + signature = hmac.new(INTERNAL_TOKEN.encode(), message, sha256).hexdigest() + return { + "X-Hotkey": hotkey, + "X-Signature": signature, + "X-Nonce": nonce, + "X-Timestamp": timestamp, + "Content-Type": "application/json", + } + + +def _checkpoint_body(submission_id: str) -> bytes: + payload = { + "submission_id": submission_id, + "attempt": 1, + "files": {"model.pt": base64.b64encode(b"weights").decode("ascii")}, + } + return json.dumps(payload).encode("utf-8") + + +def _assignment_rows(db_path: Path, submission_id: str) -> int: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT COUNT(*) FROM evaluation_assignments WHERE submission_id=?", + (submission_id,), + ).fetchone() + finally: + conn.close() + return int(row[0]) if row else 0 + + +# --- VAL-PRISM-025(a): worker hotkeys cannot publish checkpoints ---------------------------------- + + +def test_worker_hotkey_cannot_publish_checkpoint(tmp_path) -> None: + settings = _settings(tmp_path) + db_path = tmp_path / "ckpt.sqlite3" + submission_id = "sub-ckpt" + with TestClient(create_app(settings)) as client: + body = _checkpoint_body(submission_id) + # A correctly-signed upload from a non-validator (worker-bound) hotkey is rejected 403. + resp = client.post( + "/internal/v1/checkpoints", + content=body, + headers=_signed_checkpoint_headers(body, WORKER_HOTKEY), + ) + assert resp.status_code == 403, resp.text + assert "not an eligible validator" in resp.text + # No checkpoint ref was recorded (the publish body was never reached). + assert _assignment_rows(db_path, submission_id) == 0 + + +def test_validator_hotkey_still_publishes_checkpoint(tmp_path) -> None: + from prism_challenge.evaluator.checkpoint_publisher import MockCheckpointPublisher + + settings = _settings(tmp_path) + app = create_app(settings, checkpoint_publisher=MockCheckpointPublisher()) + with TestClient(app) as client: + body = _checkpoint_body("sub-ok") + resp = client.post( + "/internal/v1/checkpoints", + content=body, + headers=_signed_checkpoint_headers(body, VALIDATOR_HOTKEY), + ) + # The permitted validator passes the gate (mock publisher publishes; ref returned). + assert resp.status_code == 200, resp.text + assert resp.json()["checkpoint_ref"] + + +# --- VAL-PRISM-025(b): resume_checkpoint_ref reaches reassigned units ----------------------------- + + +def test_resume_checkpoint_ref_flows_to_reassigned_unit(tmp_path) -> None: + settings = _settings(tmp_path) + headers = {"Authorization": f"Bearer {INTERNAL_TOKEN}"} + with TestClient(create_app(settings)) as client: + app = client.app + + async def _seed() -> tuple[str, str]: + repo = app.state.repository + resumed = await repo.create_submission( + "hk-a", SubmissionCreate(code=_bundle(), filename="a.py") + ) + fresh = await repo.create_submission( + "hk-b", SubmissionCreate(code=_bundle(), filename="b.py") + ) + # A prior validator attempt published a checkpoint for the resumed submission. + await repo.record_published_checkpoint( + submission_id=resumed.id, + attempt=1, + validator_hotkey=VALIDATOR_HOTKEY, + checkpoint_ref="hf://prism/resume@rev1", + ) + return resumed.id, fresh.id + + resumed_id, fresh_id = anyio.run(_seed) + + listed = client.get("/internal/v1/work_units", headers=headers) + assert listed.status_code == 200, listed.text + by_submission = { + unit["submission_id"]: unit + for unit in listed.json()["work_units"] + if not unit.get("audit") + } + # The reassigned unit carries the resume ref; the fresh first-attempt unit carries none. + assert ( + by_submission[resumed_id]["payload"][RESUME_CHECKPOINT_PAYLOAD_KEY] + == "hf://prism/resume@rev1" + ) + assert RESUME_CHECKPOINT_PAYLOAD_KEY not in by_submission[fresh_id]["payload"] From 97df452148eb927826dbdba4f8a54d4637842259 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:19:29 +0000 Subject: [PATCH 05/14] feat(prism): NO_ACTIVE_WORKER admission rule + worker-plane flag regressions (M3 prism-proof) Adds the fail-closed submission admission rule behind worker_plane.admission_requires_worker: POST /v1/submissions and the base bridge path /internal/v1/bridge/submissions both query the master's GET /v1/workers/active?hotkey= and reject with 403 NO_ACTIVE_WORKER unless >=1 active worker is confirmed. Master unreachable/timeout/5xx folds into the same bounded 403 shape (VAL-PRISM-014/020/021). Flag OFF is byte-identical to legacy with zero master calls (VAL-PRISM-015). Scoring and get_weights/leaderboard/epoch surfaces are unchanged ON vs OFF (VAL-PRISM-016/022). --- config.example.yaml | 18 + src/prism_challenge/admission.py | 107 +++++ src/prism_challenge/app.py | 2 + src/prism_challenge/config.py | 10 + src/prism_challenge/routes.py | 2 + tests/test_prism_admission.py | 419 +++++++++++++++++++ tests/test_prism_worker_plane_regressions.py | 136 ++++++ 7 files changed, 694 insertions(+) create mode 100644 src/prism_challenge/admission.py create mode 100644 tests/test_prism_admission.py create mode 100644 tests/test_prism_worker_plane_regressions.py diff --git a/config.example.yaml b/config.example.yaml index 539627a..a7c9988 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -98,3 +98,21 @@ base_eval_task: architecture validator_hotkeys: [] validator_assignment_timeout_seconds: 900 validator_assignment_max_attempts: 3 +# Worker-plane feature block (architecture.md 3.4/3.5). OFF by default: with these defaults prism +# behaves exactly as before the compute plane (no ExecutionProof emission, no admission gate, legacy +# audit-free finalization). Env overrides use the nested delimiter, e.g. +# PRISM_WORKER_PLANE__ADMISSION_REQUIRES_WORKER=true. +worker_plane: + enabled: false + # When true, POST /v1/submissions (and the base bridge path) is rejected 403 NO_ACTIVE_WORKER + # unless the master's GET /v1/workers/active?hotkey= confirms >=1 active worker for the hotkey. + admission_requires_worker: false + # Base master coordination base URL the admission rule queries; auth reuses the bridge shared + # token. Unset while admission_requires_worker is on => admission fails closed. + master_base_url: null + admission_timeout_seconds: 5.0 + audit_rate_tier0: 0.10 + audit_rate_tier1: 0.05 + audit_rate_tier2: 0.02 + # Pinned evaluator/worker image digest (sha256:<64hex>) a tier-1 proof claim is checked against. + pinned_image_digest: null diff --git a/src/prism_challenge/admission.py b/src/prism_challenge/admission.py new file mode 100644 index 0000000..5d5738e --- /dev/null +++ b/src/prism_challenge/admission.py @@ -0,0 +1,107 @@ +"""Submission admission rule: require >=1 active worker for the submitting hotkey. + +architecture.md 3.5: when ``worker_plane.admission_requires_worker`` is on, a prism submission +(direct ``POST /v1/submissions`` AND the base bridge path ``POST /internal/v1/bridge/submissions``, +identical enforcement) is rejected with HTTP 403 code ``NO_ACTIVE_WORKER`` unless the base master's +``GET /v1/workers/active?hotkey=`` confirms at least one active worker bound to that hotkey. + +Fail-closed and bounded (VAL-PRISM-020): a master that is unreachable, times out, or returns a +non-2xx (5xx/4xx) can never confirm a worker, so the submission is rejected with the SAME 403 +``NO_ACTIVE_WORKER`` shape used for an explicit zero-worker answer -- one deterministic response, +capped by ``worker_plane.admission_timeout_seconds``. With the flag off the check is a no-op and no +master call is made, so submission behavior is byte-identical to legacy (VAL-PRISM-015). +""" + +from __future__ import annotations + +import logging + +import httpx +from fastapi import HTTPException, status + +from .config import PrismSettings + +logger = logging.getLogger(__name__) + +#: Error code returned in the 403 body when admission is denied (no confirmed active worker). +NO_ACTIVE_WORKER_CODE = "NO_ACTIVE_WORKER" + +#: Master coordination path the admission rule queries for a hotkey's active workers. +ACTIVE_WORKERS_PATH = "/v1/workers/active" + + +def _bridge_bearer(settings: PrismSettings) -> str | None: + """Reuse the prism<->master bridge shared token as the admission-query bearer, if configured.""" + try: + return settings.internal_token() + except RuntimeError: + return None + + +async def count_active_workers(settings: PrismSettings, hotkey: str) -> int | None: + """Return the number of ACTIVE workers the master reports for ``hotkey``. + + Returns ``None`` when the master cannot be consulted (unset URL, connection error, timeout, or + a non-2xx response) so the caller fails closed. The query is bounded by + ``worker_plane.admission_timeout_seconds`` so it never hangs a submission. + """ + worker_plane = settings.worker_plane + base_url = worker_plane.master_base_url + if not base_url: + logger.warning( + "admission_requires_worker is on but worker_plane.master_base_url is unset; " + "failing closed" + ) + return None + headers: dict[str, str] = {} + token = _bridge_bearer(settings) + if token: + headers["Authorization"] = f"Bearer {token}" + try: + async with httpx.AsyncClient( + base_url=base_url.rstrip("/"), + timeout=worker_plane.admission_timeout_seconds, + ) as client: + response = await client.get( + ACTIVE_WORKERS_PATH, params={"hotkey": hotkey}, headers=headers + ) + except httpx.HTTPError as exc: + logger.warning( + "admission master query failed (%s); failing closed", type(exc).__name__ + ) + return None + if response.status_code >= 400: + logger.warning( + "admission master returned HTTP %s; failing closed", response.status_code + ) + return None + try: + payload = response.json() + except ValueError: + logger.warning("admission master returned non-JSON body; failing closed") + return None + workers = payload.get("workers") if isinstance(payload, dict) else None + if not isinstance(workers, list): + logger.warning("admission master response missing a workers list; failing closed") + return None + return len(workers) + + +async def enforce_admission(settings: PrismSettings, hotkey: str) -> None: + """Reject a submission from ``hotkey`` unless the master confirms >=1 active worker. + + No-op unless ``worker_plane.admission_requires_worker`` is on (flag off => zero master calls, + legacy behavior). On denial (or any fail-closed condition) raises HTTP 403 with the + ``NO_ACTIVE_WORKER`` code so no submission row is ever created. + """ + if not settings.worker_plane.admission_requires_worker: + return + active = await count_active_workers(settings, hotkey) + if active is None or active < 1: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + { + "code": NO_ACTIVE_WORKER_CODE, + "detail": "no active worker bound to the submitting hotkey", + }, + ) diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index dbc2a61..bf4f96e 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI, Header, HTTPException, Request, status +from .admission import enforce_admission from .audit import audit_sampler_from_config, resolve_audit_unit from .auth import authenticate_internal, authenticate_validator from .config import PrismSettings, settings @@ -297,6 +298,7 @@ async def bridge_submission( ) if len(submission.code.encode()) > app_settings.max_code_bytes: raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "submission too large") + await enforce_admission(app_settings, x_base_verified_hotkey) return await repository.create_submission(x_base_verified_hotkey, submission) return app diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index ea0c5bf..17b11cd 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -19,6 +19,16 @@ class WorkerPlaneConfig(BaseModel): enabled: bool = False admission_requires_worker: bool = False + # Base master coordination base URL the admission rule queries for >=1 active worker bound to + # the submitting hotkey (``GET {master_base_url}/v1/workers/active?hotkey=``). Auth reuses the + # existing prism<->master bridge shared token as the bearer. Unset while + # ``admission_requires_worker`` is on => admission fails closed (no submission accepted), which + # is the same deterministic rejection as an explicit zero-worker answer (architecture.md 3.5). + master_base_url: str | None = None + # Bounded admission-check latency (seconds). A master that is unreachable/slow/5xx must never + # hang a submission: the query is capped here and any failure folds into the fail-closed + # NO_ACTIVE_WORKER rejection (architecture.md 3.5; VAL-PRISM-020). + admission_timeout_seconds: float = Field(default=5.0, gt=0.0) audit_rate_tier0: float = Field(default=0.10, ge=0.0, le=1.0) audit_rate_tier1: float = Field(default=0.05, ge=0.0, le=1.0) audit_rate_tier2: float = Field(default=0.02, ge=0.0, le=1.0) diff --git a/src/prism_challenge/routes.py b/src/prism_challenge/routes.py index 434db24..d55e2a1 100644 --- a/src/prism_challenge/routes.py +++ b/src/prism_challenge/routes.py @@ -17,6 +17,7 @@ ) from pydantic import ValidationError +from .admission import enforce_admission from .auth import authenticate_miner from .evaluator.architecture_report import ( generate_report_content, @@ -78,6 +79,7 @@ async def submit_model( raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, exc.errors()) from exc if len(request_body.code.encode()) > request.app.state.settings.max_code_bytes: raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "submission too large") + await enforce_admission(request.app.state.settings, hotkey) return await repository.create_submission(hotkey, request_body) diff --git a/tests/test_prism_admission.py b/tests/test_prism_admission.py new file mode 100644 index 0000000..dd72eb7 --- /dev/null +++ b/tests/test_prism_admission.py @@ -0,0 +1,419 @@ +"""Admission rule + flag-OFF regression tests (VAL-PRISM-014/015/020/021). + +A tiny threaded HTTP stub stands in for the base master's ``GET /v1/workers/active?hotkey=`` so the +tests observe exactly what prism queries and control the answer (zero/one worker, 5xx, slow, or a +closed port). Everything binds to loopback ports in the mission range 3100-3199. +""" + +from __future__ import annotations + +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +import anyio +from conftest import VALID_CODE, signed_headers +from fastapi.testclient import TestClient + +from prism_challenge.admission import NO_ACTIVE_WORKER_CODE +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig + + +def _reserve_port() -> int: + """Return a free loopback port in the mission range 3100-3199.""" + for port in range(3100, 3200): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + continue + return port + raise RuntimeError("no free port available in 3100-3199") + + +class _StubMaster: + """Threaded stub of the master's ``GET /v1/workers/active`` recording every request.""" + + def __init__(self, *, active_count: int = 0, status_code: int = 200, delay: float = 0.0): + self.active_count = active_count + self.status_code = status_code + self.delay = delay + self.requests: list[dict[str, Any]] = [] + self.port = _reserve_port() + self._server: ThreadingHTTPServer | None = None + self._thread: threading.Thread | None = None + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def __enter__(self) -> _StubMaster: + stub = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args: Any) -> None: # silence stub access logs + pass + + def do_GET(self) -> None: # noqa: N802 (http.server API) + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + stub.requests.append( + {"path": parsed.path, "hotkey": query.get("hotkey", [None])[0]} + ) + if stub.delay: + time.sleep(stub.delay) + hotkey = query.get("hotkey", [""])[0] + body = json.dumps( + { + "workers": [ + { + "worker_id": f"w{i}", + "worker_pubkey": f"pk{i}", + "miner_hotkey": hotkey, + "provider": "local", + "status": "active", + "created_at": "2026-01-01T00:00:00Z", + } + for i in range(stub.active_count) + ] + } + ).encode() + try: + self.send_response(stub.status_code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError, OSError): + pass # client (prism) already timed out and closed the connection + + self._server = ThreadingHTTPServer(("127.0.0.1", self.port), Handler) + self._server.daemon_threads = True + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + return self + + def __exit__(self, *_exc: Any) -> None: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + + +def _settings( + tmp_path: Path, + *, + admission_requires_worker: bool = False, + master_base_url: str | None = None, + admission_timeout: float = 5.0, + enabled: bool = False, + validator_hotkeys: tuple[str, ...] = (), + max_code_bytes: int = 7_500_000, +) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'prism.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + fineweb_sample_count=4, + llm_review_enabled=False, + llm_review_required=False, + distributed_contract_policy="off", + max_code_bytes=max_code_bytes, + validator_hotkeys=validator_hotkeys, + worker_plane=WorkerPlaneConfig( + enabled=enabled, + admission_requires_worker=admission_requires_worker, + master_base_url=master_base_url, + admission_timeout_seconds=admission_timeout, + ), + ) + + +def _json_body(code: str = VALID_CODE) -> bytes: + return json.dumps({"code": code, "filename": "model.py"}, separators=(",", ":")).encode() + + +def _submission_count(client: TestClient) -> int: + repo = client.app.state.repository + + async def _count() -> int: + async with repo.database.connect() as conn: + rows = await conn.execute_fetchall("SELECT COUNT(*) AS c FROM submissions") + return int(dict(rows[0])["c"]) + + return anyio.run(_count) + + +def _post_direct(client: TestClient, *, nonce: str, body: bytes | None = None) -> Any: + payload = body if body is not None else _json_body() + return client.post( + "/v1/submissions", + content=payload, + headers={ + **signed_headers("secret", payload, nonce=nonce), + "Content-Type": "application/json", + }, + ) + + +def _post_bridge( + client: TestClient, *, hotkey: str = "hk-bridge", body: bytes | None = None +) -> Any: + payload = body if body is not None else _json_body() + return client.post( + "/internal/v1/bridge/submissions", + content=payload, + headers={ + "Authorization": "Bearer secret", + "X-Base-Verified-Hotkey": hotkey, + "Content-Type": "application/json", + }, + ) + + +# --- VAL-PRISM-014: admission ON, direct route ----------------------------------------------- + + +def test_admission_on_rejects_without_active_worker(tmp_path): + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path, admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_direct(client, nonce="n1") + assert response.status_code == 403, response.text + assert response.json()["detail"]["code"] == NO_ACTIVE_WORKER_CODE + assert _submission_count(client) == 0 + # prism queried the master admission surface for the submitting hotkey + assert any( + req["path"] == "/v1/workers/active" and req["hotkey"] == "hk" + for req in stub.requests + ) + + +def test_admission_on_accepts_after_worker_active(tmp_path): + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path, admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + rejected = _post_direct(client, nonce="n1") + assert rejected.status_code == 403, rejected.text + assert _submission_count(client) == 0 + + # a worker becomes active; an identically-constructed re-signed submission is accepted + stub.active_count = 1 + accepted = _post_direct(client, nonce="n2") + assert accepted.status_code == 200, accepted.text + assert accepted.json()["hotkey"] == "hk" + assert _submission_count(client) == 1 + submission_id = accepted.json()["id"] + assert client.get(f"/v1/submissions/{submission_id}").status_code == 200 + + +# --- VAL-PRISM-015: admission flag OFF is byte-identical to legacy ---------------------------- + + +def test_admission_off_accepts_without_master_call(tmp_path): + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path, admission_requires_worker=False, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_direct(client, nonce="n1") + assert response.status_code == 200, response.text + assert response.json()["hotkey"] == "hk" + assert _submission_count(client) == 1 + assert stub.requests == [] # flag OFF => zero master calls + + +def test_admission_off_legacy_error_paths_untouched(tmp_path): + settings = _settings( + tmp_path, + admission_requires_worker=False, + validator_hotkeys=("val-hk",), + max_code_bytes=2_000, + ) + with TestClient(create_app(settings)) as client: + # invalid signature => 401 + body = _json_body() + bad_sig = client.post( + "/v1/submissions", + content=body, + headers={ + "X-Hotkey": "hk", + "X-Signature": "deadbeef", + "X-Nonce": "n-bad", + "X-Timestamp": signed_headers("secret", body)["X-Timestamp"], + "Content-Type": "application/json", + }, + ) + assert bad_sig.status_code == 401, bad_sig.text + + # validator hotkey => 403 with the LEGACY message (not NO_ACTIVE_WORKER) + val_body = _json_body() + val = client.post( + "/v1/submissions", + content=val_body, + headers={ + **signed_headers("secret", val_body, hotkey="val-hk", nonce="n-val"), + "Content-Type": "application/json", + }, + ) + assert val.status_code == 403, val.text + assert val.json()["detail"] == "validator hotkey is not allowed to submit" + + # oversized code => 413 + big = _json_body("A" * 4_000) + oversized = _post_direct(client, nonce="n-big", body=big) + assert oversized.status_code == 413, oversized.text + assert oversized.json()["detail"] == "submission too large" + + +# --- VAL-PRISM-020: admission is fail-closed and bounded when the master is unreachable ------- + + +def test_admission_fail_closed_connection_refused(tmp_path): + closed_port = _reserve_port() # nothing is listening here + settings = _settings( + tmp_path, + admission_requires_worker=True, + master_base_url=f"http://127.0.0.1:{closed_port}", + admission_timeout=2.0, + ) + with TestClient(create_app(settings)) as client: + start = time.monotonic() + response = _post_direct(client, nonce="n1") + elapsed = time.monotonic() - start + assert response.status_code == 403, response.text + assert response.json()["detail"]["code"] == NO_ACTIVE_WORKER_CODE + assert _submission_count(client) == 0 + assert elapsed < 3.0 # connection refused is immediate, well under the bound + + +def test_admission_fail_closed_timeout_is_bounded(tmp_path): + with _StubMaster(active_count=1, delay=5.0) as stub: # accepts but never answers in time + settings = _settings( + tmp_path, + admission_requires_worker=True, + master_base_url=stub.base_url, + admission_timeout=0.5, + ) + with TestClient(create_app(settings)) as client: + start = time.monotonic() + response = _post_direct(client, nonce="n1") + elapsed = time.monotonic() - start + assert response.status_code == 403, response.text + assert response.json()["detail"]["code"] == NO_ACTIVE_WORKER_CODE + assert _submission_count(client) == 0 + assert elapsed < 3.0 # bounded by admission_timeout + margin, not the 5s stub delay + + +def test_admission_fail_closed_master_5xx(tmp_path): + with _StubMaster(active_count=1, status_code=503) as stub: + settings = _settings( + tmp_path, admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_direct(client, nonce="n1") + assert response.status_code == 403, response.text + assert response.json()["detail"]["code"] == NO_ACTIVE_WORKER_CODE + assert _submission_count(client) == 0 + + +def test_admission_fail_closed_shape_is_consistent(tmp_path): + """Every fail-closed mode uses the SAME 403 NO_ACTIVE_WORKER shape (VAL-PRISM-020).""" + outcomes: list[tuple[int, str]] = [] + + # zero workers + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path / "zero", admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + r = _post_direct(client, nonce="n1") + outcomes.append((r.status_code, r.json()["detail"]["code"])) + + # 5xx + with _StubMaster(active_count=1, status_code=500) as stub: + settings = _settings( + tmp_path / "err", admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + r = _post_direct(client, nonce="n1") + outcomes.append((r.status_code, r.json()["detail"]["code"])) + + # connection refused + closed_port = _reserve_port() + settings = _settings( + tmp_path / "refused", + admission_requires_worker=True, + master_base_url=f"http://127.0.0.1:{closed_port}", + admission_timeout=1.0, + ) + with TestClient(create_app(settings)) as client: + r = _post_direct(client, nonce="n1") + outcomes.append((r.status_code, r.json()["detail"]["code"])) + + assert outcomes == [(403, NO_ACTIVE_WORKER_CODE)] * 3 + + +def test_admission_off_inert_when_master_unreachable(tmp_path): + """Flag OFF => the three unreachable master states cause zero difference and zero calls.""" + closed_port = _reserve_port() + settings = _settings( + tmp_path, + admission_requires_worker=False, + master_base_url=f"http://127.0.0.1:{closed_port}", + ) + with TestClient(create_app(settings)) as client: + response = _post_direct(client, nonce="n1") + assert response.status_code == 200, response.text + assert _submission_count(client) == 1 + + +# --- VAL-PRISM-021: the BASE bridge path enforces admission identically ----------------------- + + +def test_bridge_admission_on_rejects_without_active_worker(tmp_path): + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path, admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_bridge(client, hotkey="H") + assert response.status_code == 403, response.text + assert response.json()["detail"]["code"] == NO_ACTIVE_WORKER_CODE + assert _submission_count(client) == 0 + assert any( + req["path"] == "/v1/workers/active" and req["hotkey"] == "H" for req in stub.requests + ) + + +def test_bridge_admission_on_accepts_with_active_worker(tmp_path): + with _StubMaster(active_count=1) as stub: + settings = _settings( + tmp_path, admission_requires_worker=True, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_bridge(client, hotkey="H") + assert response.status_code == 200, response.text + assert response.json()["hotkey"] == "H" + assert _submission_count(client) == 1 + + +def test_bridge_admission_off_is_legacy(tmp_path): + with _StubMaster(active_count=0) as stub: + settings = _settings( + tmp_path, admission_requires_worker=False, master_base_url=stub.base_url + ) + with TestClient(create_app(settings)) as client: + response = _post_bridge(client, hotkey="H") + assert response.status_code == 200, response.text + assert _submission_count(client) == 1 + assert stub.requests == [] # flag OFF => zero master calls on the bridge path diff --git a/tests/test_prism_worker_plane_regressions.py b/tests/test_prism_worker_plane_regressions.py new file mode 100644 index 0000000..db7ebdf --- /dev/null +++ b/tests/test_prism_worker_plane_regressions.py @@ -0,0 +1,136 @@ +"""Worker-plane regression tests: scoring + read surfaces are unchanged ON vs OFF. + +VAL-PRISM-016: a fixture manifest finalized with ``worker_plane.enabled`` ON yields the exact same +``final_score`` as with the flag OFF (proof/plausibility/audit/admission never touch scoring). +VAL-PRISM-022: ``get_weights``/``leaderboard``/``epochs`` surfaces are byte-identical for an +identical database state whether the worker plane is ON or OFF (modulo the WeightsResponse epoch +timestamp, which is not compared here). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from conftest import VALID_CODE, signed_headers, two_script_bundle +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.sdk.executors.docker import DockerRunResult + +_INTERNAL_HEADERS = {"Authorization": "Bearer secret", "X-Base-Challenge-Slug": "prism"} + + +def _fake_run(self: Any, spec: Any, timeout_seconds: Any) -> DockerRunResult: + artifact_dir = next(mount.source for mount in spec.mounts if mount.target == "/artifacts") + manifest = { + "schema_version": "prism_run_manifest.v2", + "metrics": { + "covered_bytes": 4096, + "sum_neg_log_likelihood_nats": 2200.0, + "online_loss": [3.1, 2.9, 2.4], + "predicted_tokens": 800, + "tokens_seen": 800, + }, + } + (artifact_dir / "prism_run_manifest.v2.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + return DockerRunResult(container_name="prism-eval", stdout="", stderr="", returncode=0) + + +def _settings(db_url: str, *, enabled: bool) -> PrismSettings: + return PrismSettings( + database_url=db_url, + shared_token="secret", + allow_insecure_signatures=True, + fineweb_sample_count=4, + llm_review_enabled=False, + llm_review_required=False, + distributed_contract_policy="off", + worker_plane=WorkerPlaneConfig(enabled=enabled), + ) + + +def _submit_and_process(client: TestClient) -> str: + payload = {"code": two_script_bundle(arch_code=VALID_CODE), "filename": "project.zip"} + body = json.dumps(payload, separators=(",", ":")).encode() + response = client.post( + "/v1/submissions", + content=body, + headers={**signed_headers("secret", body), "Content-Type": "application/json"}, + ) + assert response.status_code == 200, response.text + submission_id = response.json()["id"] + process = client.post( + "/internal/v1/worker/process-next", headers={"Authorization": "Bearer secret"} + ) + assert process.status_code == 200, process.text + assert process.json()["submission_id"] == submission_id + return submission_id + + +def _read_surfaces(client: TestClient) -> dict[str, Any]: + weights = client.get("/internal/v1/get_weights", headers=_INTERNAL_HEADERS).json()["weights"] + leaderboard = client.get("/v1/leaderboard").json() + epochs = client.get("/v1/epochs").json() + current = client.get("/v1/epochs/current").json() + return { + "weights": weights, + "leaderboard": leaderboard, + "epochs": epochs, + "current": current, + } + + +def test_final_score_identical_worker_plane_on_off(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr("prism_challenge.evaluator.container.DockerExecutor.run", _fake_run) + + on_url = f"sqlite+aiosqlite:///{tmp_path / 'on.sqlite3'}" + with TestClient(create_app(_settings(on_url, enabled=True))) as client: + sub_id = _submit_and_process(client) + on_status = client.get(f"/v1/submissions/{sub_id}").json() + + off_url = f"sqlite+aiosqlite:///{tmp_path / 'off.sqlite3'}" + with TestClient(create_app(_settings(off_url, enabled=False))) as client: + sub_id = _submit_and_process(client) + off_status = client.get(f"/v1/submissions/{sub_id}").json() + + assert on_status["status"] == "completed" + assert off_status["status"] == "completed" + assert on_status["final_score"] == off_status["final_score"] + + +def test_read_surfaces_identical_worker_plane_on_off(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr("prism_challenge.evaluator.container.DockerExecutor.run", _fake_run) + db_url = f"sqlite+aiosqlite:///{tmp_path / 'shared.sqlite3'}" + + # Author a crowned-architecture state under the flag ON, then read every surface. + with TestClient(create_app(_settings(db_url, enabled=True))) as client: + _submit_and_process(client) + on_surfaces = _read_surfaces(client) + + # Read the SAME database state under the flag OFF: the worker plane never touches these reads. + with TestClient(create_app(_settings(db_url, enabled=False))) as client: + off_surfaces = _read_surfaces(client) + + assert on_surfaces == off_surfaces + assert on_surfaces["weights"] != {} # a crowned architecture is paid + assert on_surfaces["leaderboard"]["entries"] # non-empty leaderboard + + +def test_read_surfaces_identical_worker_plane_on_off_burn(tmp_path: Path) -> None: + db_url = f"sqlite+aiosqlite:///{tmp_path / 'burn.sqlite3'}" + + with TestClient(create_app(_settings(db_url, enabled=True))) as client: + on_surfaces = _read_surfaces(client) + + with TestClient(create_app(_settings(db_url, enabled=False))) as client: + off_surfaces = _read_surfaces(client) + + # BURN state: no positive q_arch_best => empty weights, empty leaderboard, identical ON/OFF. + assert on_surfaces == off_surfaces + assert on_surfaces["weights"] == {} + assert on_surfaces["leaderboard"]["entries"] == [] From 12a2c0b31ba9d128407737291c4274580b607b76 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:35:38 +0000 Subject: [PATCH 06/14] feat(prism): finalize worker-plane results from forwarded manifest without re-execution (M3 prism-finalization) Under worker_plane.enabled, ingest_work_unit_result now finalizes the submission score from the forwarded, verified+reconciled prism_run_manifest.v2 via a new PrismWorker.finalize_worker_result that takes NO GPU lease and NEVER constructs the evaluator (no evaluator.evaluate / _evaluate_within_wall_time / _augment_with_heldout). It reuses the _finalize_container_score tail with skip_heldout=True so the score is prequential-bpb-only and the master-only secret val split (base_eval_val_data_dir) is never read (architecture.md 4). Dedup/conflict and proof+plausibility gates run before finalization exactly as-is; flag OFF keeps the legacy process_submission re-execution path byte-identical. Fulfills VAL-FINAL-001..004. --- src/prism_challenge/evaluator/scoring.py | 21 +- src/prism_challenge/ingestion.py | 29 +- src/prism_challenge/queue.py | 63 ++- tests/test_prism_audit_scheduler.py | 24 +- tests/test_prism_plausibility.py | 38 +- tests/test_prism_result_ingestion.py | 25 +- tests/test_prism_worker_plane_finalization.py | 455 ++++++++++++++++++ 7 files changed, 631 insertions(+), 24 deletions(-) create mode 100644 tests/test_prism_worker_plane_finalization.py diff --git a/src/prism_challenge/evaluator/scoring.py b/src/prism_challenge/evaluator/scoring.py index 59d3a30..80f7dd3 100644 --- a/src/prism_challenge/evaluator/scoring.py +++ b/src/prism_challenge/evaluator/scoring.py @@ -245,7 +245,7 @@ def dedupe_best_per_hotkey(rows: Iterable[LeaderboardRow]) -> list[LeaderboardRo def score_prequential_bpb( - manifest: Mapping[str, Any], *, sane_max: float = BPB_SANE_MAX + manifest: Mapping[str, Any], *, sane_max: float = BPB_SANE_MAX, skip_heldout: bool = False ) -> PrequentialBpbScore: """Compute the prequential bits-per-byte score from the challenge-owned v2 manifest. @@ -253,6 +253,12 @@ def score_prequential_bpb( numerator is the token-weighted online (predict-then-train) negative log-likelihood the challenge captured itself. Raises ``ScoreValidationError`` for a degenerate (zero-coverage, non-finite, or out-of-band) run so it never collapses into a fabricated/0-that-ranks score. + + ``skip_heldout`` grades on prequential bpb ALONE (delta ``None``, no tie-break, no memorization + penalty) regardless of any held-out fields the manifest carries. The worker plane passes it + because the held-out delta needs the master-only secret val split, which never reaches a + miner-funded worker (architecture.md 4); a forwarded manifest therefore scores bpb-only and a + fabricated held-out field can never move the score. """ metrics = manifest.get("metrics") if not isinstance(metrics, Mapping): @@ -282,7 +288,18 @@ def score_prequential_bpb( if bool(anti_cheat.get("nan_inf_detected", False)): flags.append("nan_inf_detected") anti_cheat_multiplier = 0.0 if step0_anomaly else 1.0 - heldout = _read_heldout(manifest, metrics, anti_cheat, train_bpb=bpb) + heldout = ( + _HeldoutView( + delta=None, + val_bpb_trained=None, + val_bpb_random_init=None, + gap=None, + memorization_flag=False, + penalty=1.0, + ) + if skip_heldout + else _read_heldout(manifest, metrics, anti_cheat, train_bpb=bpb) + ) if heldout.memorization_flag: flags.append("memorization_gap") # The held-out delta refines ranking only as a NEAR-TIE tie-breaker (bounded additive term, diff --git a/src/prism_challenge/ingestion.py b/src/prism_challenge/ingestion.py index cd6bf3b..95686cc 100644 --- a/src/prism_challenge/ingestion.py +++ b/src/prism_challenge/ingestion.py @@ -16,11 +16,13 @@ downgrade recorded, so the audit scheduler samples at the honest rate. Only a fully verified result is finalized, and finalization is idempotent: the first accepted -delivery finalizes the submission through the CAS-guarded :meth:`PrismWorker.process_submission` -path (a no-op when the executing worker already finalized it against a shared store), while a -duplicate delivery of the same manifest is a no-op and a CONFLICTING delivery (a different -``manifest_sha256`` for an already-accepted unit) is rejected -- never overwriting the stored score -(VAL-PRISM-017). +delivery finalizes the submission from the FORWARDED, verified+reconciled manifest via the +CAS-guarded :meth:`PrismWorker.finalize_worker_result` path (worker plane on) -- scoring the run +WITHOUT re-executing the evaluator, since the heavy GPU work already ran on the miner-funded worker +(architecture.md 4). A duplicate delivery of the same manifest is a no-op and a CONFLICTING delivery +(a different ``manifest_sha256`` for an already-accepted unit) is rejected -- never overwriting the +stored score (VAL-PRISM-017). With the worker plane OFF, finalization falls back to the legacy +in-process re-execution path (:meth:`PrismWorker.process_submission`), byte-for-byte unchanged. """ from __future__ import annotations @@ -81,7 +83,9 @@ class ResultIngestionError(Exception): * ``manifest_tampered`` -- the forwarded manifest does not hash to ``manifest_sha256`` (VAL-PRISM-007a); * ``signature_invalid`` -- the worker signature does not verify for this unit - (VAL-PRISM-007b/c). + (VAL-PRISM-007b/c); + * ``manifest_missing`` -- the worker plane is on but no run manifest was forwarded, so there is + nothing to finalize from without re-executing (VAL-FINAL-001). """ def __init__(self, reason: str, message: str = "") -> None: @@ -278,7 +282,18 @@ async def ingest_work_unit_result( wall_clock_budget_seconds=float(worker.settings.base_eval_hard_timeout_seconds), ) - result_id = await worker.process_submission(submission_id) + if worker.settings.worker_plane.enabled: + # Worker plane: finalize from the forwarded, verified+reconciled manifest WITHOUT + # re-executing the evaluator (the heavy GPU work already ran on the miner-funded worker). + if manifest is None: + raise ResultIngestionError( + "manifest_missing", + "worker-plane finalization requires the forwarded run manifest", + ) + result_id = await worker.finalize_worker_result(submission_id, dict(manifest)) + else: + # Flag OFF: legacy in-process re-execution finalization, byte-for-byte unchanged. + result_id = await worker.process_submission(submission_id) submission_status = await repository.submission_status(submission_id) await repository.record_work_unit_result( work_unit_id=work_unit_id, diff --git a/src/prism_challenge/queue.py b/src/prism_challenge/queue.py index 6196bed..0c377d2 100644 --- a/src/prism_challenge/queue.py +++ b/src/prism_challenge/queue.py @@ -161,6 +161,62 @@ async def process_submission( return None return await self._process_claimed(submission, resume_checkpoint_ref=resume_checkpoint_ref) + async def finalize_worker_result( + self, submission_id: str, manifest: dict[str, Any] + ) -> str | None: + """Finalize a submission from a forwarded worker manifest WITHOUT re-executing. + + The heavy GPU evaluation already ran on the miner-funded worker; ingestion has already + verified+reconciled the run (proof + plausibility). This claims the pending submission (a + CAS, so a duplicate delivery or an already-finalized submission is a no-op returning + ``None``) and finalizes it from the forwarded ``prism_run_manifest.v2`` alone: the + challenge-owned prequential bpb (``score_prequential_bpb``) with the deterministic + source-static tail -- the AST anti-cheat multiplier (``evaluate_anti_cheat`` over the + submission SOURCE) and the static fingerprints/arch_hash/name from the component review. It + takes NO GPU lease and NEVER constructs the evaluator (no ``evaluator.evaluate`` / + ``_evaluate_within_wall_time`` / ``_augment_with_heldout``). The held-out delta is SKIPPED + (``skip_heldout=True``) so the score is bpb-only and the master-only secret val split + (``base_eval_val_data_dir``) is never read (architecture.md 4). + """ + submission = await self.repository.claim_submission(submission_id) + if submission is None: + return None + code = str(submission["code"]) + filename = str(submission.get("filename") or "model.py") + raw_metadata = submission.get("metadata") + metadata = cast(dict[str, Any], raw_metadata) if isinstance(raw_metadata, dict) else {} + hotkey = str(submission.get("hotkey") or "") + try: + snapshot = self._snapshot_from_submission(code, filename, metadata) + component_review = self._component_review(snapshot) + code_for_eval = self._entrypoint_code( + snapshot, component_review.components.entrypoint + ) + arch_hash = component_review.fingerprints.family_hash or sha256( + code_for_eval.encode() + ).hexdigest() + arch_name = architecture_name(component_review.components) + previous = await self.repository.previous_codes(submission_id) + anti = evaluate_anti_cheat( + code_for_eval, + previous, + allowed_import_roots=self._local_import_roots(snapshot), + ) + except Exception as exc: + await self._fail_submission(submission_id, str(exc)) + return submission_id + await self._finalize_container_score( + submission_id=submission_id, + arch_hash=arch_hash, + anti=anti, + manifest=manifest, + hotkey=hotkey, + fingerprints=component_review.fingerprints, + name=arch_name, + skip_heldout=True, + ) + return submission_id + async def _process_claimed( self, submission: dict[str, Any], *, resume_checkpoint_ref: str | None = None ) -> str: @@ -796,6 +852,7 @@ async def _finalize_container_score( hotkey: str = "", fingerprints: PrismComponentFingerprints | None = None, name: str | None = None, + skip_heldout: bool = False, ) -> None: """Finalize a container run using the CHALLENGE-OWNED prequential bits-per-byte score. @@ -809,9 +866,13 @@ async def _finalize_container_score( and the ``training_variants`` row (keyed by ``(architecture_id, training_hash)``), and persists the loss curve + reconciled compute block into ``submission_curves`` so the data is centrally queryable (none of these are inputs to the score). + + ``skip_heldout`` forces the bpb-only scoring path (no held-out delta tie-break); the worker + plane sets it so a forwarded worker manifest is graded on prequential bpb alone without ever + needing the master-only secret val split (architecture.md 4). """ try: - score = score_prequential_bpb(manifest) + score = score_prequential_bpb(manifest, skip_heldout=skip_heldout) except ScoreValidationError as exc: await self._fail_submission(submission_id, f"prequential scoring failed: {exc}") return diff --git a/tests/test_prism_audit_scheduler.py b/tests/test_prism_audit_scheduler.py index c6e283a..676aa3e 100644 --- a/tests/test_prism_audit_scheduler.py +++ b/tests/test_prism_audit_scheduler.py @@ -24,6 +24,7 @@ import base64 import io +import math import sqlite3 import zipfile from pathlib import Path @@ -139,10 +140,29 @@ def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettin def _manifest(marker: str = "v2") -> dict[str, Any]: + """A plausible AND scoreable v2 manifest (worker-plane finalization scores from it directly).""" + + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] return { "schema_version": "prism_run_manifest.v2", - "metrics": {"prequential_bpb": 1.23, "marker": marker}, - "steps": [{"step": 0, "loss": 3.0}, {"step": 1, "loss": 2.0}], + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, } diff --git a/tests/test_prism_plausibility.py b/tests/test_prism_plausibility.py index e91e12b..411dabb 100644 --- a/tests/test_prism_plausibility.py +++ b/tests/test_prism_plausibility.py @@ -22,6 +22,8 @@ from prism_challenge.app import create_app from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.anti_cheat import evaluate_anti_cheat +from prism_challenge.evaluator.components import architecture_name from prism_challenge.evaluator.mock_reexec import cpu_reexec_run from prism_challenge.ingestion import ingest_work_unit_result from prism_challenge.models import SubmissionCreate @@ -68,6 +70,7 @@ def _plausible_manifest() -> dict[str, Any]: "prequential_bpb": 1.23, "bits_per_byte": 1.23, "covered_bytes": 4096, + "sum_neg_log_likelihood_nats": 3000.0, }, "score": { "schema": "prism_score.v2", @@ -394,18 +397,14 @@ def _wallclock() -> dict[str, Any]: assert await app.state.repository.submission_status(submission_id) == "pending" -async def test_ingestion_accepts_plausible_manifest_identically_to_legacy(tmp_path, monkeypatch): - data_dir = _stage_train(tmp_path) - monkeypatch.setattr( - "prism_challenge.evaluator.container.DockerExecutor.run", - cpu_reexec_run(train_data_dir=data_dir), - ) +async def test_ingestion_accepts_plausible_manifest_identically_to_legacy(tmp_path): settings = _settings( tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) ) app = await _make_app(settings) signer = worker_signer_from_key(WORKER_KEY) db_path = tmp_path / "coord.sqlite3" + worker = app.state.worker # Path A: finalize a plausible worker result through the plausibility-gated ingestion path. gated_id = await _seed(app) @@ -413,7 +412,7 @@ async def test_ingestion_accepts_plausible_manifest_identically_to_legacy(tmp_pa before = copy.deepcopy(manifest) proof = _proof_dict(signer, gated_id, manifest) outcome = await ingest_work_unit_result( - worker=app.state.worker, + worker=worker, work_unit_id=gated_id, submission_ref="hk-owner", result=_result(proof, manifest), @@ -423,13 +422,32 @@ async def test_ingestion_accepts_plausible_manifest_identically_to_legacy(tmp_pa assert await app.state.repository.submission_status(gated_id) == "completed" gated_score = _score(db_path, gated_id) assert gated_score is not None - # The forwarded manifest / score inputs are passed through UNCHANGED. + # The forwarded manifest / score inputs are passed through UNCHANGED (plausibility only READS). assert manifest == before - # Path B: finalize an identical submission through the legacy in-process path directly. + # Path B: feed the SAME manifest to the legacy finalization tail directly (VAL-PRISM-010). legacy_id = await _seed(app) - await app.state.worker.process_submission(legacy_id) + submission = await worker.repository.claim_submission(legacy_id) + assert submission is not None + snapshot = worker._snapshot_from_submission(str(submission["code"]), "project.zip", {}) + review = worker._component_review(snapshot) + code_for_eval = worker._entrypoint_code(snapshot, review.components.entrypoint) + anti = evaluate_anti_cheat( + code_for_eval, + await worker.repository.previous_codes(legacy_id), + allowed_import_roots=worker._local_import_roots(snapshot), + ) + await worker._finalize_container_score( + submission_id=legacy_id, + arch_hash=review.fingerprints.family_hash, + anti=anti, + manifest=copy.deepcopy(manifest), + hotkey="hk-owner", + fingerprints=review.fingerprints, + name=architecture_name(review.components), + ) legacy_score = _score(db_path, legacy_id) assert legacy_score is not None assert gated_score == pytest.approx(legacy_score) + diff --git a/tests/test_prism_result_ingestion.py b/tests/test_prism_result_ingestion.py index 2a4fdfc..f543180 100644 --- a/tests/test_prism_result_ingestion.py +++ b/tests/test_prism_result_ingestion.py @@ -10,6 +10,7 @@ import base64 import io +import math import sqlite3 import zipfile from pathlib import Path @@ -119,10 +120,30 @@ def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettin def _manifest(marker: str = "v2") -> dict[str, Any]: + """A plausible AND scoreable v2 manifest (worker-plane finalization scores from it directly).""" + + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] return { "schema_version": "prism_run_manifest.v2", - "metrics": {"prequential_bpb": 1.23, "marker": marker}, - "steps": [{"step": 0, "loss": 3.0}, {"step": 1, "loss": 2.0}], + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "prequential_bpb": 1.23, + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, } diff --git a/tests/test_prism_worker_plane_finalization.py b/tests/test_prism_worker_plane_finalization.py new file mode 100644 index 0000000..1213d39 --- /dev/null +++ b/tests/test_prism_worker_plane_finalization.py @@ -0,0 +1,455 @@ +"""Worker-plane LIGHT finalization: score from the forwarded manifest, no re-execution. + +Covers the mission finalization assertions (architecture.md 4): + +* VAL-FINAL-001 - with the worker plane ON, ingesting a verified+plausible worker result finalizes + the submission from the FORWARDED ``prism_run_manifest.v2`` (prequential bpb + the deterministic + source-static tail); the heavy evaluator (``_evaluate_within_wall_time`` / ``evaluator.evaluate``) + and the GPU lease are NEVER invoked during ingest finalization. +* VAL-FINAL-002 - the worker-plane score is bpb-only (held-out delta absent/None even when the + forwarded manifest fabricates one) and the master-only secret val split + (``base_eval_val_data_dir``) is never read/required (no evaluator is constructed). +* VAL-FINAL-003 - with the worker plane OFF, ingestion falls back to the legacy + ``process_submission`` re-execution path and produces a byte-identical score. +* VAL-FINAL-004 - the canonical manifest round-trip is stable (on-disk bytes hash == the + ingestion-recomputed canonical hash), so a genuine manifest is never falsely rejected. + +Offline, no GPU: worker-plane finalization scores from the forwarded manifest directly and the +flag-off regression drives the CPU re-exec seam the other coordination tests use. Proofs are signed +with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.evaluator.schemas import RUN_MANIFEST_V2_FILENAME +from prism_challenge.evaluator.scoring import score_prequential_bpb +from prism_challenge.ingestion import ResultIngestionError, ingest_work_unit_result +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + canonical_manifest_json, + compute_manifest_sha256, + worker_signer_from_key, +) +from prism_challenge.queue import PrismWorker + +WORKER_KEY = "//WorkerFinalize" + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings( + tmp_path: Path, + *, + worker_plane: WorkerPlaneConfig, + base_eval_val_data_dir: str | None = None, +) -> PrismSettings: + overrides: dict[str, Any] = {} + if base_eval_val_data_dir is not None: + overrides["base_eval_val_data_dir"] = base_eval_val_data_dir + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=worker_plane, + **overrides, + ) + + +def _scoreable_manifest( + marker: str = "v2", *, heldout_delta: float | None = None +) -> dict[str, Any]: + """A plausible AND scoreable v2 manifest (positive finite bpb, decreasing loss).""" + + covered_bytes = 4096 + sum_nll_nats = 900.0 + baseline = math.log(50257) + online_loss = [10.0, 6.0, 3.0, 2.0] + metrics: dict[str, Any] = { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": sum_nll_nats, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": baseline, + "marker": marker, + } + if heldout_delta is not None: + # A worker CANNOT compute a legitimate held-out delta (no secret val split); a + # fabricated one must never move the finalized score. + metrics["heldout_delta"] = heldout_delta + metrics["val_bpb_trained"] = 0.20 + metrics["val_bpb_random_init"] = 0.20 + heldout_delta + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": metrics, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _proof_dict(signer, unit_id: str, manifest: dict[str, Any], **overrides: Any) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + payload = proof.model_dump(mode="json") + payload.update(overrides) + return payload + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any] | None) -> dict[str, Any]: + result: dict[str, Any] = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + } + if manifest is not None: + result[MANIFEST_PAYLOAD_KEY] = manifest + return result + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _score_row(db_path: Path, submission_id: str) -> tuple[float, dict[str, Any]] | None: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score, metrics FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + if row is None: + return None + metrics = json.loads(row[1]) if row[1] else {} + return float(row[0]), metrics + + +def _final_score(db_path: Path, submission_id: str) -> float | None: + row = _score_row(db_path, submission_id) + return row[0] if row is not None else None + + +# --- VAL-FINAL-001: finalize from the forwarded manifest, no re-execution ------------------------ + + +async def test_worker_plane_finalizes_from_manifest_without_reexecution(tmp_path, monkeypatch): + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + # The heavy evaluator seam and the GPU lease scheduler must NOT be touched during finalization. + def _boom_eval(*_a: Any, **_k: Any): + raise AssertionError("evaluator was invoked during worker-plane ingest finalization") + + def _boom_lease(*_a: Any, **_k: Any): + raise AssertionError("a GPU lease was taken during worker-plane ingest finalization") + + monkeypatch.setattr(PrismWorker, "_evaluate_within_wall_time", _boom_eval) + monkeypatch.setattr("prism_challenge.queue.GpuLeaseScheduler", _boom_lease) + + submission_id = await _seed(app) + manifest = _scoreable_manifest() + proof = _proof_dict(signer, submission_id, manifest) + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert outcome.status == "accepted" + assert outcome.finalized is True + assert await app.state.repository.submission_status(submission_id) == "completed" + + # The persisted score equals the score computed independently from the forwarded manifest + # (anti-cheat multiplier is 1.0 for a first submission with no prior codes). + stored = _final_score(db_path, submission_id) + expected = score_prequential_bpb(manifest).final_score + assert stored is not None + assert stored == pytest.approx(expected) + + +async def test_worker_plane_requires_forwarded_manifest(tmp_path): + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + submission_id = await _seed(app) + manifest = _scoreable_manifest() + # A proof-only delivery (no manifest to score from) cannot be finalized without re-executing. + proof = _proof_dict(signer, submission_id, manifest) + with pytest.raises(ResultIngestionError) as exc: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, None), + ) + assert exc.value.reason == "manifest_missing" + assert _final_score(db_path, submission_id) is None + assert await app.state.repository.submission_status(submission_id) == "pending" + + +# --- VAL-FINAL-002: held-out delta skipped; secret split never read ------------------------------ + + +async def test_worker_plane_skips_heldout_and_never_reads_secret_split(tmp_path, monkeypatch): + settings = _settings( + tmp_path, + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + base_eval_val_data_dir="/nonexistent/secret/val-split", + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + # If ANY code path constructs the evaluator (the only reader of base_eval_val_data_dir, via + # _augment_with_heldout) or takes a GPU lease, fail loudly. + def _boom_eval(*_a: Any, **_k: Any): + raise AssertionError("evaluator constructed during worker-plane finalization") + + def _boom_lease(*_a: Any, **_k: Any): + raise AssertionError("a GPU lease was taken during worker-plane finalization") + + monkeypatch.setattr(PrismWorker, "_evaluate_within_wall_time", _boom_eval) + monkeypatch.setattr("prism_challenge.queue.GpuLeaseScheduler", _boom_lease) + + # A manifest that FABRICATES a held-out delta must be scored bpb-only anyway. + forged = await _seed(app) + forged_manifest = _scoreable_manifest("forged", heldout_delta=5.0) + forged_proof = _proof_dict(signer, forged, forged_manifest) + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=forged, + submission_ref="hk-owner", + result=_result(forged_proof, forged_manifest), + ) + forged_row = _score_row(db_path, forged) + assert forged_row is not None + forged_score, forged_metrics = forged_row + + # bpb-only: equals score_prequential_bpb with the held-out delta skipped ... + bpb_only = score_prequential_bpb(forged_manifest, skip_heldout=True).final_score + assert forged_score == pytest.approx(bpb_only) + # ... and is NOT the delta-inclusive score the tie-break would have produced. + assert forged_score != pytest.approx(score_prequential_bpb(forged_manifest).final_score) + # No held-out contribution is recorded in the finalized metrics. + assert "heldout_delta" not in forged_metrics + assert "held_out_delta" not in forged_metrics + + # A submission WITHOUT any held-out field scores identically to the fabricated-delta one. + plain = await _seed(app) + plain_manifest = _scoreable_manifest("plain") + plain_proof = _proof_dict(signer, plain, plain_manifest) + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=plain, + submission_ref="hk-owner", + result=_result(plain_proof, plain_manifest), + ) + assert _final_score(db_path, plain) == pytest.approx(forged_score) + + +# --- VAL-FINAL-003: flag OFF => legacy re-execution finalization, byte-identical ----------------- + + +async def test_flag_off_uses_legacy_reexecution_finalization(tmp_path, monkeypatch): + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + + # With the flag OFF the worker-plane finalizer must never be used. + def _boom_finalize(*_a: Any, **_k: Any): + raise AssertionError("worker-plane finalizer used while the flag was OFF") + + monkeypatch.setattr(PrismWorker, "finalize_worker_result", _boom_finalize) + + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=False, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + # Path A: ingest with the flag OFF -> legacy in-process re-execution finalization. + ingest_id = await _seed(app) + manifest = _scoreable_manifest() + proof = _proof_dict(signer, ingest_id, manifest) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=ingest_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + ) + assert outcome.finalized is True + assert await app.state.repository.submission_status(ingest_id) == "completed" + ingest_score = _final_score(db_path, ingest_id) + assert ingest_score is not None + + # Path B: the legacy path directly on an identical submission -> byte-identical score. + legacy_id = await _seed(app) + await app.state.worker.process_submission(legacy_id) + legacy_score = _final_score(db_path, legacy_id) + assert legacy_score is not None + assert ingest_score == pytest.approx(legacy_score) + + +# --- VAL-FINAL-004: canonical manifest round-trip is stable -------------------------------------- + + +def test_manifest_canonicalization_round_trip_stable(tmp_path): + manifest = _scoreable_manifest() + path = tmp_path / RUN_MANIFEST_V2_FILENAME + # Emission persists the canonical form (sort_keys=True, indent=2). + path.write_text(json.dumps(manifest, sort_keys=True, indent=2), encoding="utf-8") + on_disk_bytes = path.read_bytes() + on_disk_sha = hashlib.sha256(on_disk_bytes).hexdigest() + + # Read -> parse -> re-serialize canonically -> identical bytes/sha. + parsed = json.loads(on_disk_bytes.decode("utf-8")) + reserialized = json.dumps(parsed, sort_keys=True, indent=2).encode("utf-8") + assert hashlib.sha256(reserialized).hexdigest() == on_disk_sha + + # The ingestion-side recompute agrees with the emission-side on-disk hash ... + assert compute_manifest_sha256(parsed) == on_disk_sha + # ... and is key-order-insensitive (a manifest built in a different order hashes identically). + shuffled = dict(reversed(list(manifest.items()))) + assert compute_manifest_sha256(shuffled) == on_disk_sha + + +async def test_genuine_manifest_never_falsely_rejected_as_tampered(tmp_path): + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + submission_id = await _seed(app) + manifest = _scoreable_manifest() + # Emission-side: sign over the canonical hash, persist to disk exactly as the runner/host would. + path = tmp_path / RUN_MANIFEST_V2_FILENAME + path.write_text(canonical_manifest_json(manifest), encoding="utf-8") + # Ingestion-side: read + parse the on-disk manifest, then verify + finalize. + parsed = json.loads(path.read_text(encoding="utf-8")) + proof = _proof_dict(signer, submission_id, parsed) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, parsed), + ) + # A genuine manifest is accepted, not falsely flagged manifest_tampered. + assert outcome.status == "accepted" + assert outcome.finalized is True + assert _final_score(db_path, submission_id) is not None From 702d925249d9d3402a69ff51b58e87fb5829f933 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:16:11 +0000 Subject: [PATCH 07/14] feat(prism): execute sampled audits in the validator cycle + salt the audit sampler (M3 prism-finalization) Wire the audit-only validator cycle end-to-end: enumerate pending audit: units (is_audit_unit_id), replay the audited submission's evaluation deterministically for a fresh manifest_sha256, and resolve via resolve_audit_unit -- matching replay leaves the finalized score untouched, divergent replay invalidates it and records a worker_fault. dispatch_assignment routes on unit type so validators run audits while workers still execute primaries + emit proofs under the flag; the autonomous run_validator_cycle is audit-only when the flag is on and legacy primary execution when off. Mix a server-side secret WorkerPlaneConfig.audit_salt (repr=False) into the AuditSampler seed so selection is unpredictable from submission_id yet reproducible for a fixed salt, preserving per-tier rates (VAL-FINAL-005/006). --- config.example.yaml | 4 + src/prism_challenge/audit.py | 54 ++- src/prism_challenge/config.py | 5 + src/prism_challenge/db.py | 14 + src/prism_challenge/queue.py | 93 ++++ src/prism_challenge/repository.py | 68 +++ src/prism_challenge/validator_dispatch.py | 81 +++- src/prism_challenge/validator_executor.py | 128 ++++- tests/test_prism_audit_execution_wiring.py | 521 +++++++++++++++++++++ 9 files changed, 952 insertions(+), 16 deletions(-) create mode 100644 tests/test_prism_audit_execution_wiring.py diff --git a/config.example.yaml b/config.example.yaml index a7c9988..3c90255 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -114,5 +114,9 @@ worker_plane: audit_rate_tier0: 0.10 audit_rate_tier1: 0.05 audit_rate_tier2: 0.02 + # Server-side SECRET salt mixed into the audit sampler seed so audit selection cannot be predicted + # from the public submission_id alone, yet stays reproducible for a fixed salt (VAL-FINAL-006). + # Kept out of the config repr. Leave null to sample from the seed only (legacy behaviour). + audit_salt: null # Pinned evaluator/worker image digest (sha256:<64hex>) a tier-1 proof claim is checked against. pinned_image_digest: null diff --git a/src/prism_challenge/audit.py b/src/prism_challenge/audit.py index bed29cd..3aa95d0 100644 --- a/src/prism_challenge/audit.py +++ b/src/prism_challenge/audit.py @@ -104,13 +104,16 @@ class AuditSampler: The sampled fraction of each tier converges to its configured rate; a rate of ``0.0`` yields exactly zero samples for that tier and ``>= 1.0`` samples every one. Sampling is a pure function - of ``seed`` and the per-result key, so it is reproducible and order-insensitive. + of ``seed``, the server-side secret ``salt`` and the per-result key, so it is reproducible and + order-insensitive. Mixing the secret ``salt`` in makes selection unpredictable from the public + ``submission_id`` alone yet reproducible for a fixed salt (architecture.md 3.4; VAL-FINAL-006). """ audit_rate_tier0: float = 0.10 audit_rate_tier1: float = 0.05 audit_rate_tier2: float = 0.02 seed: int = 0 + salt: str = "" def rate_for_tier(self, tier: int) -> float: """The configured audit rate for an EFFECTIVE tier (unknown tiers fall back to tier 0).""" @@ -149,20 +152,30 @@ def decide( ) def _uniform(self, key: str) -> float: - """A deterministic uniform draw in ``[0, 1)`` keyed on ``(seed, key)``.""" + """A deterministic uniform draw in ``[0, 1)`` keyed on ``(seed, salt, key)``. - digest = hashlib.sha256(f"{self.seed}:{key}".encode()).digest() + The secret ``salt`` is folded into the hashed material so the draw for a given public + ``submission_id`` cannot be reproduced without it (VAL-FINAL-006). + """ + + digest = hashlib.sha256(f"{self.seed}:{self.salt}:{key}".encode()).digest() return int.from_bytes(digest[:8], "big") / float(1 << 64) def audit_sampler_from_config(worker_plane: WorkerPlaneConfig, *, seed: int = 0) -> AuditSampler: - """Build an :class:`AuditSampler` from the prism ``worker_plane`` audit-rate config.""" + """Build an :class:`AuditSampler` from the prism ``worker_plane`` audit-rate config. + + The server-side secret ``audit_salt`` is mixed into the sampler seed so audit selection is + unpredictable from the public ``submission_id`` yet reproducible for a fixed salt + (VAL-FINAL-006). + """ return AuditSampler( audit_rate_tier0=worker_plane.audit_rate_tier0, audit_rate_tier1=worker_plane.audit_rate_tier1, audit_rate_tier2=worker_plane.audit_rate_tier2, seed=seed, + salt=worker_plane.audit_salt or "", ) @@ -184,6 +197,19 @@ async def record_audit_resolution( async def invalidate_submission_score(self, submission_id: str, *, reason: str) -> bool: ... + async def get_work_unit_result(self, work_unit_id: str) -> dict[str, object] | None: ... + + async def record_worker_fault( + self, + *, + audit_unit_id: str, + submission_id: str, + worker_pubkey: str | None, + audited_manifest_sha256: str, + replay_manifest_sha256: str, + reason: str, + ) -> None: ... + @dataclass(frozen=True) class AuditResolution: @@ -278,6 +304,7 @@ async def resolve_audit_unit( ) audited_hash = str(unit["audited_manifest_sha256"]) + assert replay_manifest_sha256 is not None # narrowed: inconclusive returned above matches = replay_manifest_sha256 == audited_hash invalidated = False if matches: @@ -290,6 +317,25 @@ async def resolve_audit_unit( submission_id, reason=f"audit invalidated: manifest mismatch (audit_unit={audit_unit_id})", ) + # The authoritative validator replay diverged from the audited worker manifest: the worker + # that produced it lied. Record a worker_fault against it (architecture.md 4; + # VAL-FINAL-005). The faulty worker's pubkey is the one recorded on the audited primary + # result; a missing result row leaves it null but still records the fault. + origin_work_unit_id = str(unit["origin_work_unit_id"]) + worker_result = await repository.get_work_unit_result(origin_work_unit_id) + worker_pubkey = ( + str(worker_result["worker_pubkey"]) + if worker_result is not None and worker_result.get("worker_pubkey") is not None + else None + ) + await repository.record_worker_fault( + audit_unit_id=audit_unit_id, + submission_id=submission_id, + worker_pubkey=worker_pubkey, + audited_manifest_sha256=audited_hash, + replay_manifest_sha256=replay_manifest_sha256, + reason="audit manifest mismatch", + ) await repository.record_audit_resolution( audit_unit_id=audit_unit_id, status=new_status, diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index 17b11cd..ccd8ad5 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -32,6 +32,11 @@ class WorkerPlaneConfig(BaseModel): audit_rate_tier0: float = Field(default=0.10, ge=0.0, le=1.0) audit_rate_tier1: float = Field(default=0.05, ge=0.0, le=1.0) audit_rate_tier2: float = Field(default=0.02, ge=0.0, le=1.0) + # Server-side SECRET salt mixed into the audit sampler seed so audit selection cannot be + # predicted from the public ``submission_id`` alone (a different salt selects a different set), + # while staying reproducible for a fixed salt and preserving the per-tier rates (architecture.md + # 3.4; VAL-FINAL-006). Kept out of the config repr like every other secret. + audit_salt: str | None = Field(default=None, repr=False) # sr25519 signing key (URI ``//Name`` / mnemonic / seed) for the worker that emits # ExecutionProofs. This is the worker's OWN key, injected by the worker agent -- NEVER a # master-side secret. Unset -> prism emits no signed proof (the base worker plane may still diff --git a/src/prism_challenge/db.py b/src/prism_challenge/db.py index de97634..91a0b9b 100644 --- a/src/prism_challenge/db.py +++ b/src/prism_challenge/db.py @@ -195,6 +195,12 @@ "created_at TEXT NOT NULL, updated_at TEXT NOT NULL);" "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" + "CREATE TABLE IF NOT EXISTS worker_faults (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, audit_unit_id TEXT NOT NULL," + "submission_id TEXT NOT NULL," + "worker_pubkey TEXT, audited_manifest_sha256 TEXT NOT NULL," + "replay_manifest_sha256 TEXT NOT NULL, reason TEXT NOT NULL, created_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_worker_faults_submission ON worker_faults(submission_id);" ) @@ -344,6 +350,14 @@ async def _run_migrations(conn: aiosqlite.Connection) -> None: "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" ) + await conn.executescript( + "CREATE TABLE IF NOT EXISTS worker_faults (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, audit_unit_id TEXT NOT NULL," + "submission_id TEXT NOT NULL, worker_pubkey TEXT," + "audited_manifest_sha256 TEXT NOT NULL, replay_manifest_sha256 TEXT NOT NULL," + "reason TEXT NOT NULL, created_at TEXT NOT NULL);" + "CREATE INDEX IF NOT EXISTS idx_worker_faults_submission ON worker_faults(submission_id);" + ) await conn.executescript( "CREATE TABLE IF NOT EXISTS gpu_leases (" "id TEXT PRIMARY KEY, submission_id TEXT NOT NULL, job_id TEXT, target_id TEXT," diff --git a/src/prism_challenge/queue.py b/src/prism_challenge/queue.py index 0c377d2..c858600 100644 --- a/src/prism_challenge/queue.py +++ b/src/prism_challenge/queue.py @@ -43,6 +43,7 @@ targets_from_settings, ) from .models import SubmissionStatus +from .proof import compute_manifest_sha256, read_manifest_sha256 from .repository import PrismRepository, now_iso from .sdk.executors.docker import DockerExecutor, DockerLimits, DockerMount, DockerRunSpec @@ -217,6 +218,98 @@ async def finalize_worker_result( ) return submission_id + async def replay_audit_manifest_sha256( + self, submission_id: str, *, resume_checkpoint_ref: str | None = None + ) -> str | None: + """Re-execute a finalized submission's evaluation for an audit; return its manifest sha. + + Audits are the sampled minority the validator bears (architecture.md 3.5): the whole + evaluation is replayed on the validator's OWN broker to obtain an authoritative + ``prism_run_manifest.v2`` to compare against the audited worker manifest. This path is + VERIFY-ONLY -- it never claims the submission, writes a score, records an eval job, or + changes the submission status -- so a passing audit leaves the finalized result untouched. + The already-passed static / LLM gates are skipped; only the container re-execution is + repeated (the honest run is deterministic, so an honest worker's hash reproduces). Returns + ``None`` on any replay failure, resolving the audit inconclusive rather than confirming it. + """ + if self.execution_backend not in CONTAINER_EXECUTION_BACKENDS: + return None + submission = await self.repository.submission_execution_row(submission_id) + if submission is None: + return None + code = str(submission["code"]) + filename = str(submission.get("filename") or "model.py") + metadata = cast(dict[str, Any], submission["metadata"]) + code_hash = str(submission.get("code_hash") or sha256(code.encode()).hexdigest()) + try: + snapshot = self._snapshot_from_submission(code, filename, metadata) + component_review = self._component_review(snapshot) + code_for_eval = self._entrypoint_code(snapshot, component_review.components.entrypoint) + arch_hash = component_review.fingerprints.family_hash or sha256( + code_for_eval.encode() + ).hexdigest() + execution_mode = execution_mode_from_value(metadata.get("execution_mode")) + except Exception: + return None + runtime_config = await self.repository.runtime_config(self.settings, official=True) + score_eligible = metadata.get("score_eligible") + scheduler = GpuLeaseScheduler( + self.repository.database, targets_from_settings(self.settings, runtime_config) + ) + lease = await scheduler.enqueue_or_allocate( + lease_request_from_runtime( + submission_id=submission_id, + job_id=None, + runtime_policy=runtime_config, + mode=execution_mode.value, + score_eligible=bool(score_eligible) if score_eligible is not None else None, + ) + ) + if not lease.active: + return None + effective_settings = self.settings.model_copy( + update={ + "base_eval_gpu_count": lease.gpu_count, + "base_eval_gpu_type": runtime_config.gpu_policy.gpu_type, + "base_eval_gpu_server": lease.target_server, + "base_eval_gpu_device_ids": lease.device_ids, + } + ) + evaluator = self._evaluator_factory(effective_settings, self.ctx) + if self._checkpoint_publisher is not None and evaluator._checkpoint_publisher is None: + evaluator._checkpoint_publisher = self._checkpoint_publisher + attempt = ( + await self.repository.container_job_attempt_count(submission_id, self.execution_backend) + + 1 + ) + try: + result = await self._evaluate_within_wall_time( + evaluator, + submission_id=submission_id, + code=code_for_eval, + code_hash=code_hash, + arch_hash=arch_hash, + files=snapshot.files, + components=component_review.components, + gpu_lease=lease, + execution_mode=execution_mode, + attempt=attempt, + resume_checkpoint_ref=resume_checkpoint_ref, + ) + except Exception: + return None + finally: + await asyncio.to_thread(evaluator.reap_job, submission_id) + await scheduler.release_for_submission(submission_id, "audit replay finished") + if not _is_v2_run_manifest(result.run_manifest): + return None + if result.run_manifest_path: + try: + return read_manifest_sha256(result.run_manifest_path) + except OSError: + pass + return compute_manifest_sha256(cast(dict[str, Any], result.run_manifest)) + async def _process_claimed( self, submission: dict[str, Any], *, resume_checkpoint_ref: str | None = None ) -> str: diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index d400d75..9ec124d 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -935,6 +935,25 @@ async def claim_submission(self, submission_id: str) -> dict[str, object] | None data["metadata"] = metadata if isinstance(metadata, dict) else {} return data + async def submission_execution_row(self, submission_id: str) -> dict[str, object] | None: + """Return the full submission row (code + metadata) for a re-execution, ignoring status. + + Unlike :meth:`claim_submission` this takes NO claim and mutates nothing, so an already + terminal (``completed``/``failed``) submission can be replayed for a validator audit without + disturbing its finalized record (architecture.md 3.5; VAL-FINAL-005). ``None`` when absent. + """ + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT * FROM submissions WHERE id=?", (submission_id,) + ) + row_list = list(rows) + if not row_list: + return None + data = dict(cast(Any, row_list[0])) + metadata = loads(str(data.get("metadata", "{}"))) + data["metadata"] = metadata if isinstance(metadata, dict) else {} + return data + async def list_pending_submissions(self) -> list[dict[str, object]]: """Return submissions awaiting re-execution (one prism work unit each), oldest first.""" async with self.database.connect() as conn: @@ -1186,6 +1205,55 @@ async def record_audit_resolution( ), ) + async def record_worker_fault( + self, + *, + audit_unit_id: str, + submission_id: str, + worker_pubkey: str | None, + audited_manifest_sha256: str, + replay_manifest_sha256: str, + reason: str, + ) -> None: + """Record a fault against the worker whose manifest diverged from the validator replay. + + Written on an audit MISMATCH (architecture.md 4; VAL-FINAL-005): the audited submission's + finalized result named ``worker_pubkey`` as its producer, and the authoritative replay + proved that manifest wrong. The fault is observational (it never mutates the submission or + any worker record); it is the durable record that this worker lied on this audited unit. + """ + async with self.database.connect() as conn: + await conn.execute( + "INSERT INTO worker_faults(" + "audit_unit_id, submission_id, worker_pubkey, audited_manifest_sha256," + "replay_manifest_sha256, reason, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + audit_unit_id, + submission_id, + worker_pubkey, + audited_manifest_sha256, + replay_manifest_sha256, + reason, + now_iso(), + ), + ) + + async def list_worker_faults( + self, *, submission_id: str | None = None + ) -> list[dict[str, object]]: + """Return recorded worker faults (optionally scoped to one submission), oldest first.""" + async with self.database.connect() as conn: + if submission_id is None: + rows = await conn.execute_fetchall( + "SELECT * FROM worker_faults ORDER BY id" + ) + else: + rows = await conn.execute_fetchall( + "SELECT * FROM worker_faults WHERE submission_id=? ORDER BY id", + (submission_id,), + ) + return [dict(cast(Any, row)) for row in rows] + async def invalidate_submission_score(self, submission_id: str, *, reason: str) -> bool: """Invalidate a finalized submission's score and recompute crown/weights aggregates. diff --git a/src/prism_challenge/validator_dispatch.py b/src/prism_challenge/validator_dispatch.py index 806eb6c..7d5f826 100644 --- a/src/prism_challenge/validator_dispatch.py +++ b/src/prism_challenge/validator_dispatch.py @@ -28,11 +28,12 @@ from typing import Any from .app import create_app +from .audit import is_audit_unit_id from .config import PrismSettings from .config import settings as default_settings from .evaluator.checkpoint_publisher import CheckpointPublisher from .proof import MANIFEST_PAYLOAD_KEY, PROOF_PAYLOAD_KEY -from .validator_executor import run_validator_cycle +from .validator_executor import run_primary_execution_cycle, run_validator_audit_cycle CHALLENGE_SLUG = "prism" @@ -61,14 +62,36 @@ async def dispatch_assignment( settings: PrismSettings | None = None, checkpoint_publisher: CheckpointPublisher | None = None, ) -> dict[str, Any]: - """Run a pulled prism assignment's GPU re-execution on the validator's broker. + """Run a pulled prism assignment on the caller's own broker (architecture.md 4). - Returns the cycle counts (pulled/executed/skipped/completed_submissions) for - the platform validator agent to post back to the master. + Routes on the UNIT TYPE so the same dispatch entrypoint serves both roles that reuse it: + + * an ``audit:`` unit (only assigned to a VALIDATOR, and only when the worker plane is on) is + replayed + resolved (see :func:`_dispatch_audit_only`); the finalized score is untouched on a + matching replay and invalidated + fault-recorded on a divergent one (VAL-FINAL-005); + * any other (PRIMARY) unit is GPU-re-executed via :func:`run_primary_execution_cycle` -- this is + the miner-funded worker path when the flag is on (it emits the ExecutionProof base reconciles) + and the legacy validator re-execution when the flag is off. + + The autonomous validator cycle's audit-only restriction lives in :func:`run_validator_cycle`; + here a primary unit always executes because the base assignment plane only ever routes a primary + unit to a worker (flag on) or a legacy validator (flag off), never to an audit-only validator. + Returns the cycle counts for the platform agent to post back to the master. """ + base_settings = settings or default_settings + if base_settings.worker_plane.enabled and is_audit_unit_id(work_unit_id): + return await _dispatch_audit_only( + work_unit_id=work_unit_id, + broker_url=broker_url, + broker_token=broker_token, + broker_token_file=broker_token_file, + settings=base_settings, + checkpoint_publisher=checkpoint_publisher, + ) + effective = gateway_scoped_settings( - settings or default_settings, + base_settings, payload, broker_url=broker_url, broker_token=broker_token, @@ -78,7 +101,9 @@ async def dispatch_assignment( database = app.state.database await database.init() try: - summary = await run_validator_cycle(worker=app.state.worker, work_unit_ids=[work_unit_id]) + summary = await run_primary_execution_cycle( + worker=app.state.worker, work_unit_ids=[work_unit_id] + ) finally: await database.close() result: dict[str, Any] = { @@ -100,6 +125,50 @@ async def dispatch_assignment( return result +async def _dispatch_audit_only( + *, + work_unit_id: str, + broker_url: str, + broker_token: str | None, + broker_token_file: str | None, + settings: PrismSettings, + checkpoint_publisher: CheckpointPublisher | None, +) -> dict[str, Any]: + """Audit-only dispatch: replay + resolve a sampled ``audit:`` unit (VAL-FINAL-005). + + Audits skip the LLM review, so this path needs NO master gateway scoped token (unlike the + primary path) -- only the validator's own broker for the deterministic re-execution. + """ + + effective = settings.model_copy( + update={ + "docker_broker_url": broker_url, + "docker_broker_token": broker_token, + "docker_broker_token_file": broker_token_file, + } + ) + app = create_app(effective, checkpoint_publisher=checkpoint_publisher) + database = app.state.database + await database.init() + try: + summary = await run_validator_audit_cycle( + worker=app.state.worker, work_unit_ids=[work_unit_id] + ) + finally: + await database.close() + invalidated = sum(1 for resolution in summary.audits if resolution.invalidated) + return { + "pulled": summary.pulled, + "executed": summary.executed, + "skipped": summary.skipped, + "completed_submissions": [], + "is_audit": is_audit_unit_id(work_unit_id), + "audits_resolved": len(summary.audits), + "audits_invalidated": invalidated, + "audit_results": [resolution.to_response() for resolution in summary.audits], + } + + def gateway_scoped_settings( settings: PrismSettings, payload: Mapping[str, Any], diff --git a/src/prism_challenge/validator_executor.py b/src/prism_challenge/validator_executor.py index d092cdf..2fef820 100644 --- a/src/prism_challenge/validator_executor.py +++ b/src/prism_challenge/validator_executor.py @@ -17,11 +17,12 @@ import json import logging import os -from collections.abc import Iterable, Mapping +from collections.abc import Awaitable, Callable, Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path from typing import Any +from .audit import AuditResolution, is_audit_unit_id, resolve_audit_unit from .coordination import ( PRISM_DEFAULT_CONCURRENCY, PRISM_WORK_UNIT_CAPABILITY, @@ -39,6 +40,11 @@ logger = logging.getLogger(__name__) +#: Replays an audited submission's evaluation and returns its fresh canonical manifest sha256 +#: (``None`` on an inconclusive replay failure). Defaults to +#: :meth:`PrismWorker.replay_audit_manifest_sha256`; injectable so tests pin a deterministic hash. +AuditReplayFn = Callable[[str], Awaitable[str | None]] + #: Submission statuses at which a prism work unit is terminal (no re-execution, no re-dispatch). TERMINAL_SUBMISSION_STATUSES = frozenset({"completed", "failed", "rejected"}) @@ -74,6 +80,9 @@ class PrismValidatorCycleSummary: execution_proofs: dict[str, dict[str, Any]] = field(default_factory=dict) #: Run manifests backing the emitted proofs, keyed by ``work_unit_id`` (empty when off). execution_manifests: dict[str, dict[str, Any]] = field(default_factory=dict) + #: Audit resolutions produced this cycle when the worker plane is ON (audit-only cycle); empty + #: on the legacy flag-off primary path (VAL-FINAL-005). + audits: tuple[AuditResolution, ...] = () async def execute_work_unit( @@ -187,15 +196,62 @@ async def run_validator_cycle( max_concurrency: int = PRISM_DEFAULT_CONCURRENCY, proof_signer: WorkerSigner | None = None, proof_env: Mapping[str, str] | None = None, + audit_replay: AuditReplayFn | None = None, +) -> PrismValidatorCycleSummary: + """Run one decentralized validator cycle. + + This is the autonomous validator entry point (the base validator agent's cycle). With the worker + plane ON it is AUDIT-ONLY: workers execute the primary gpu submissions (base assignment plane + + forwarding + light finalization), so the validator cycle NEVER pulls or executes a primary + submission -- it only pulls the sampled ``audit:`` units, replays each deterministically, and + resolves them (architecture.md 4; VAL-FINAL-005). With the flag OFF it is the legacy + primary-execution cycle (:func:`run_primary_execution_cycle`), unchanged (no audit units exist). + + NOTE: worker-plane PRIMARY execution (a miner-funded worker running an assigned gpu unit + the + ExecutionProof emission that feeds base reconciliation) goes through + :func:`run_primary_execution_cycle` directly (via ``dispatch_assignment`` routing on the unit + type), NOT through this flag-gated entry -- so a worker still executes primaries while the flag + is on, and only the VALIDATOR cycle is audit-only. + """ + + if worker.settings.worker_plane.enabled: + return await run_validator_audit_cycle( + worker=worker, work_unit_ids=work_unit_ids, audit_replay=audit_replay + ) + return await run_primary_execution_cycle( + worker=worker, + work_unit_ids=work_unit_ids, + capabilities=capabilities, + in_flight=in_flight, + max_concurrency=max_concurrency, + proof_signer=proof_signer, + proof_env=proof_env, + ) + + +async def run_primary_execution_cycle( + *, + worker: PrismWorker, + work_unit_ids: Iterable[str] | None = None, + capabilities: Iterable[str] = (PRISM_WORK_UNIT_CAPABILITY,), + in_flight: int | None = None, + max_concurrency: int = PRISM_DEFAULT_CONCURRENCY, + proof_signer: WorkerSigner | None = None, + proof_env: Mapping[str, str] | None = None, ) -> PrismValidatorCycleSummary: - """Run one decentralized validator cycle: pull -> execute (own broker) -> post. + """Pull -> execute (own broker) -> post the caller's assigned PRIMARY prism units. Pulls the caller's assigned, capability-matched prism units (at most - ``max_concurrency - in_flight`` of them, so a busy validator runs one submission at a time), - executes each on the validator's own broker, and reports which submissions completed. The pull - and assignment are execution-free; only :func:`execute_work_unit` dispatches the broker. + ``max_concurrency - in_flight`` of them, so a busy executor runs one submission at a time), + re-executes each on the caller's own broker, and reports which submissions completed. The pull + and assignment are execution-free; only :func:`execute_work_unit` dispatches the broker. A + successful worker-plane finalization emits an ExecutionProof per unit (architecture.md 3.4). - ``in_flight`` defaults to the validator's REAL in-flight draw (the count of currently-running + This is the shared primary-execution path for BOTH a legacy validator (worker plane off) and a + miner-funded worker running an assigned gpu unit (worker plane on); the audit-only restriction + lives in :func:`run_validator_cycle`, not here. + + ``in_flight`` defaults to the caller's REAL in-flight draw (the count of currently-running submissions) so the concurrency-1 cap is enforced against reality rather than a static zero; a caller may override it (e.g. tests pinning a specific value). """ @@ -237,3 +293,63 @@ async def run_validator_cycle( execution_proofs=execution_proofs, execution_manifests=execution_manifests, ) + + +async def run_validator_audit_cycle( + *, + worker: PrismWorker, + work_unit_ids: Iterable[str] | None = None, + audit_replay: AuditReplayFn | None = None, +) -> PrismValidatorCycleSummary: + """Execute the sampled prism AUDIT units assigned to this validator (architecture.md 3.5). + + Enumerates the pending ``audit:`` units (``list_pending_audit_units``, each id recognised via + :func:`~prism_challenge.audit.is_audit_unit_id`), restricted to the caller's assigned + ``work_unit_ids`` when given (``None`` = every pending audit). For each, the audited + submission's evaluation is replayed deterministically to obtain a fresh manifest sha256, and the + replay result is resolved through :func:`~prism_challenge.audit.resolve_audit_unit` (the + ``POST /internal/v1/audit_units/{id}/result`` target): a MATCHING hash passes (finalized score + untouched); a DIVERGENT hash invalidates the score and records a ``worker_fault``; a replay + failure resolves inconclusive (re-audited within bounds). This cycle NEVER executes a primary + submission (VAL-FINAL-005). ``audit_replay`` defaults to the real container replay; tests inject + a deterministic hash. + """ + + replay = audit_replay if audit_replay is not None else worker.replay_audit_manifest_sha256 + wanted = set(work_unit_ids) if work_unit_ids is not None else None + pending = await worker.repository.list_pending_audit_units() + resolutions: list[AuditResolution] = [] + pulled = 0 + executed = 0 + for row in pending: + audit_unit_id = str(row["audit_unit_id"]) + if not is_audit_unit_id(audit_unit_id): + continue + if wanted is not None and audit_unit_id not in wanted: + continue + pulled += 1 + submission_id = str(row["submission_id"]) + replay_hash: str | None + error: str | None = None + try: + replay_hash = await replay(submission_id) + except Exception as exc: # noqa: BLE001 - a replay failure is an inconclusive audit + logger.warning("audit replay failed for %s: %s", audit_unit_id, exc) + replay_hash = None + error = str(exc) + executed += 1 + resolution = await resolve_audit_unit( + worker.repository, + audit_unit_id=audit_unit_id, + replay_manifest_sha256=replay_hash, + failed=replay_hash is None, + error=error, + ) + resolutions.append(resolution) + return PrismValidatorCycleSummary( + pulled=pulled, + executed=executed, + skipped=0, + completed_submissions=(), + audits=tuple(resolutions), + ) diff --git a/tests/test_prism_audit_execution_wiring.py b/tests/test_prism_audit_execution_wiring.py new file mode 100644 index 0000000..92ea1c5 --- /dev/null +++ b/tests/test_prism_audit_execution_wiring.py @@ -0,0 +1,521 @@ +"""Validator audit-execution wiring + salted audit sampler (architecture.md 3.4/3.5). + +Covers the mission finalization assertions this feature delivers: + +* VAL-FINAL-005 - with the worker plane ON the validator cycle is AUDIT-ONLY: it pulls a sampled + ``audit:`` unit, replays the audited submission's evaluation to obtain a fresh manifest hash, and + resolves it through ``resolve_audit_unit`` (the ``POST /internal/v1/audit_units/{id}/result`` + target). A MATCHING replay leaves the finalized score untouched; a DIVERGENT replay invalidates + the score AND records a ``worker_fault``. Under the flag the validator cycle NEVER executes a + primary submission; with the flag OFF the cycle is the legacy primary-execution path (no audits). +* VAL-FINAL-006 - the audit sampler mixes a server-side secret salt into its seed so selection is + unpredictable from the public ``submission_id`` alone (a different salt selects a different set) + yet reproducible for a fixed salt, and the per-tier rates are preserved statistically. + +Offline, no GPU: finalization + the real replay run through the CPU re-exec seam the other +coordination tests use, and proofs are signed with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import io +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.audit import ( + AUDIT_STATUS_MISMATCH, + AUDIT_STATUS_PASSED, + AuditSampler, + audit_sampler_from_config, + audit_unit_id_for, +) +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) +from prism_challenge.queue import PrismWorker +from prism_challenge.validator_dispatch import dispatch_assignment +from prism_challenge.validator_executor import ( + run_validator_audit_cycle, + run_validator_cycle, +) + +WORKER_KEY = "//WorkerAuditExec" +EPOCH_SECONDS = 60 +BROKER_URL = "http://broker-val:8082" + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'audit_exec.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + epoch_seconds=EPOCH_SECONDS, + worker_plane=worker_plane, + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _result(signer, unit_id: str, manifest: dict[str, Any], **extra: Any) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + return { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof.model_dump(mode="json"), + MANIFEST_PAYLOAD_KEY: manifest, + **extra, + } + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _always() -> AuditSampler: + return AuditSampler(audit_rate_tier0=1.0, audit_rate_tier1=1.0, audit_rate_tier2=1.0) + + +async def _finalize_and_sample(app, signer, *, hotkey: str = "hk-owner") -> tuple[str, dict, Any]: + """Finalize a worker result (worker plane) and force it sampled -> a pending audit unit.""" + from prism_challenge.ingestion import ingest_work_unit_result + + submission_id = await _seed(app, hotkey) + manifest = _manifest() + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=hotkey, + result=_result(signer, submission_id, manifest), + audit_sampler=_always(), + ) + return submission_id, manifest, outcome + + +def _score(db_path: Path, submission_id: str): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return row[0] if row else None + + +# --- VAL-FINAL-006: salted audit sampler --------------------------------------------------------- + + +def test_audit_salt_shifts_selection_but_is_reproducible_for_fixed_salt() -> None: + ids = [f"submission-{i}" for i in range(400)] + + def _select(salt: str) -> list[bool]: + sampler = AuditSampler(audit_rate_tier0=0.10, seed=7, salt=salt) + return [sampler.should_sample(work_unit_id=i, effective_tier=0) for i in ids] + + no_salt = _select("") + salt_a1 = _select("SERVER-SECRET-A") + salt_a2 = _select("SERVER-SECRET-A") + salt_b = _select("SERVER-SECRET-B") + + # A fixed salt reproduces the same selection ... + assert salt_a1 == salt_a2 + # ... but a different salt selects a different set (unpredictable from submission_id alone) ... + assert salt_a1 != salt_b + # ... and the salted selection is not the same as the public (submission_id-only) one. + assert salt_a1 != no_salt + + +def test_audit_salt_preserves_per_tier_rates_statistically() -> None: + n = 6000 + + def _bound(p: float) -> float: + return 4.0 * (p * (1.0 - p) / n) ** 0.5 + + for salt in ("", "salt-one", "another-secret-salt"): + sampler = AuditSampler( + audit_rate_tier0=0.10, audit_rate_tier1=0.05, audit_rate_tier2=0.02, seed=99, salt=salt + ) + for tier, rate in ((0, 0.10), (1, 0.05), (2, 0.02)): + hits = sum( + sampler.should_sample(work_unit_id=f"{salt}-t{tier}-{i}", effective_tier=tier) + for i in range(n) + ) + assert abs(hits / n - rate) < _bound(rate), (salt, tier, hits / n, rate) + + +def test_audit_sampler_from_config_mixes_secret_salt_repr_hidden() -> None: + ids = [f"unit-{i}" for i in range(300)] + cfg_a = WorkerPlaneConfig(enabled=True, audit_salt="TOP-SECRET-SALT") + cfg_b = WorkerPlaneConfig(enabled=True, audit_salt="OTHER-SECRET-SALT") + cfg_none = WorkerPlaneConfig(enabled=True) + + sel_a = [ + audit_sampler_from_config(cfg_a).should_sample(work_unit_id=i, effective_tier=0) + for i in ids + ] + sel_a_again = [ + audit_sampler_from_config(cfg_a).should_sample(work_unit_id=i, effective_tier=0) + for i in ids + ] + sel_b = [ + audit_sampler_from_config(cfg_b).should_sample(work_unit_id=i, effective_tier=0) + for i in ids + ] + sel_none = [ + audit_sampler_from_config(cfg_none).should_sample(work_unit_id=i, effective_tier=0) + for i in ids + ] + + assert sel_a == sel_a_again # reproducible for a fixed salt + assert sel_a != sel_b # a different salt shifts the selected set + assert sel_a != sel_none # unpredictable from submission_id (the public value) alone + # The salt is a server-side SECRET: it never appears in the config repr. + assert "TOP-SECRET-SALT" not in repr(cfg_a) + assert "audit_salt" not in repr(cfg_a) + + +# --- VAL-FINAL-005: validator audit cycle (injectable replay for deterministic hashes) ----------- + + +async def test_matching_replay_passes_and_leaves_score_untouched(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit_exec.sqlite3" + + submission_id, manifest, outcome = await _finalize_and_sample(app, signer) + assert outcome.audit_unit_id == audit_unit_id_for(submission_id) + score_before = _score(db_path, submission_id) + assert score_before is not None + + # A validator replay reproducing the audited manifest hash: the audit passes, score untouched. + async def _replay(sub_id: str) -> str: + assert sub_id == submission_id + return compute_manifest_sha256(manifest) + + summary = await run_validator_audit_cycle( + worker=app.state.worker, + work_unit_ids=[outcome.audit_unit_id], + audit_replay=_replay, + ) + assert summary.pulled == 1 + assert summary.executed == 1 + assert len(summary.audits) == 1 + assert summary.audits[0].status == AUDIT_STATUS_PASSED + assert summary.audits[0].invalidated is False + assert _score(db_path, submission_id) == pytest.approx(score_before) + assert await repository.submission_status(submission_id) == "completed" + assert await repository.list_worker_faults(submission_id=submission_id) == [] + + +async def test_divergent_replay_invalidates_score_and_records_worker_fault(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit_exec.sqlite3" + + submission_id, _, outcome = await _finalize_and_sample(app, signer) + assert _score(db_path, submission_id) is not None + + async def _replay(_sub_id: str) -> str: + return "f" * 64 # authoritative replay diverges from the worker's manifest hash + + summary = await run_validator_audit_cycle( + worker=app.state.worker, + work_unit_ids=[outcome.audit_unit_id], + audit_replay=_replay, + ) + assert summary.audits[0].status == AUDIT_STATUS_MISMATCH + assert summary.audits[0].invalidated is True + # Score invalidated + submission dropped from the leaderboard. + assert _score(db_path, submission_id) is None + assert await repository.submission_status(submission_id) == "failed" + + # A worker_fault is recorded against the divergent (lying) worker for the audited unit. + faults = await repository.list_worker_faults(submission_id=submission_id) + assert len(faults) == 1 + fault = faults[0] + assert fault["submission_id"] == submission_id + assert fault["worker_pubkey"] == signer.worker_pubkey + assert fault["audit_unit_id"] == outcome.audit_unit_id + assert fault["replay_manifest_sha256"] == "f" * 64 + + +async def test_validator_cycle_is_audit_only_when_flag_on(tmp_path, monkeypatch) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + # A pending PRIMARY submission that the validator cycle must NOT execute while the flag is on. + primary_id = await _seed(app, "hk-primary") + + # An independent finalized+sampled submission (its audit unit is the only work the cycle bears). + audited_id, manifest, outcome = await _finalize_and_sample(app, signer, hotkey="hk-audited") + + # If the cycle ever tried to execute a primary submission, these would fire. + def _boom_container(*_a: Any, **_k: Any): + raise AssertionError("validator cycle executed a primary submission while the flag was ON") + + monkeypatch.setattr(PrismWorker, "_evaluate_within_wall_time", _boom_container) + + async def _replay(sub_id: str) -> str: + assert sub_id == audited_id + return compute_manifest_sha256(manifest) + + summary = await run_validator_cycle(worker=app.state.worker, audit_replay=_replay) + + # The primary was never pulled/executed; it stays pending, unscored. + assert summary.executed == 0 or all( + res.audit_unit_id != primary_id for res in summary.audits + ) + assert await repository.submission_status(primary_id) == "pending" + # The sampled audit was executed + resolved. + assert len(summary.audits) == 1 + assert summary.audits[0].status == AUDIT_STATUS_PASSED + assert await repository.submission_status(audited_id) == "completed" + + +async def test_validator_cycle_flag_off_executes_primary_and_runs_no_audit( + tmp_path, monkeypatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings(tmp_path, worker_plane=WorkerPlaneConfig(enabled=False)) + app = await _make_app(settings) + repository = app.state.repository + db_path = tmp_path / "audit_exec.sqlite3" + + primary_id = await _seed(app, "hk-legacy") + + # Even if a stray audit row exists, the flag-off cycle never touches audits. + await repository.create_audit_unit( + submission_id=primary_id, + origin_work_unit_id=primary_id, + audited_manifest_sha256="a" * 64, + effective_tier=0, + ) + + def _boom_replay(*_a: Any, **_k: Any): + raise AssertionError("flag-off validator cycle attempted an audit replay") + + monkeypatch.setattr(PrismWorker, "replay_audit_manifest_sha256", _boom_replay) + + summary = await run_validator_cycle(worker=app.state.worker) + # Legacy primary execution: the submission is re-executed and finalized, no audit resolutions. + assert summary.executed == 1 + assert primary_id in summary.completed_submissions + assert summary.audits == () + assert _score(db_path, primary_id) is not None + + +async def test_dispatch_audit_unit_real_replay_diverges_invalidates_and_faults( + tmp_path, monkeypatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "audit_exec.sqlite3" + + # Finalize from a hand-crafted worker manifest; the honest replay produces the REAL runner + # manifest which differs, so the audit must catch the divergence. + submission_id, _, outcome = await _finalize_and_sample(app, signer) + await app.state.database.close() + assert _score(db_path, submission_id) is not None + + # The base validator agent dispatches the pulled audit unit here. The audit payload carries NO + # gateway token (audits skip the LLM review), so the audit path must not require one. + result = await dispatch_assignment( + work_unit_id=outcome.audit_unit_id, + payload={"audit": True, "audited_submission_id": submission_id}, + broker_url=BROKER_URL, + settings=settings, + ) + assert result["audits_resolved"] == 1 + assert result["audits_invalidated"] == 1 + + app2 = await _make_app(settings) + repository = app2.state.repository + assert _score(db_path, submission_id) is None + assert await repository.submission_status(submission_id) == "failed" + faults = await repository.list_worker_faults(submission_id=submission_id) + assert len(faults) == 1 + assert faults[0]["worker_pubkey"] == signer.worker_pubkey + + +def test_audit_result_route_records_worker_fault_on_mismatch(tmp_path, monkeypatch) -> None: + import anyio + from fastapi.testclient import TestClient + + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + signer = worker_signer_from_key(WORKER_KEY) + headers = {"Authorization": "Bearer secret"} + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + with TestClient(create_app(settings)) as client: + app = client.app + + async def _drive() -> tuple[str, str]: + sid, _, out = await _finalize_and_sample(app, signer) + return sid, out.audit_unit_id + + submission_id, audit_unit_id = anyio.run(_drive) + + # A validator posts its divergent replay hash to the internal audit route -> resolve_audit. + resolved = client.post( + f"/internal/v1/audit_units/{audit_unit_id}/result", + json={"manifest_sha256": "e" * 64}, + headers=headers, + ) + assert resolved.status_code == 200, resolved.text + assert resolved.json()["status"] == AUDIT_STATUS_MISMATCH + assert resolved.json()["invalidated"] is True + + async def _faults() -> list[dict[str, Any]]: + return await app.state.repository.list_worker_faults(submission_id=submission_id) + + faults = anyio.run(_faults) + assert len(faults) == 1 + assert faults[0]["worker_pubkey"] == signer.worker_pubkey From 6cc8f7416a094c538c8cd8b0ee6ee7c0a54a98f8 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:26:52 +0000 Subject: [PATCH 08/14] fix(prism): advance family owner_hotkey to survivor on invalidation (VAL-FINAL-007) On audit-mismatch invalidation, _recompute_family_after_invalidation now advances architecture_families.owner_hotkey (and owner_submission_id) to the surviving best submission's owner alongside canonical_submission_id/q_arch_best, so a proven-faulty family creator loses the 0.60 architecture emission share that get_weights rewards and the surviving co-owner is paid. The single-owner no-survivor burn (q_arch_best -> 0.0) and the normal (non-invalidated) persistent-crown-pays-creator semantics are unchanged. Adds a multi-owner-same-arch regression test proving the faulty owner loses the share and get_weights pays the survivor. --- src/prism_challenge/repository.py | 19 ++++- tests/test_prism_audit_scheduler.py | 125 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index 9ec124d..cec0b1f 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -1304,7 +1304,7 @@ async def _recompute_family_after_invalidation( return architecture_id = str(fam_list[0]["id"]) best_rows = await conn.execute_fetchall( - "SELECT s.id AS sid, sc.final_score AS fs FROM submissions s " + "SELECT s.id AS sid, s.hotkey AS owner, sc.final_score AS fs FROM submissions s " "JOIN scores sc ON sc.submission_id=s.id " "WHERE s.arch_hash=? AND s.status=? " "ORDER BY sc.final_score DESC, s.created_at ASC, s.id ASC LIMIT 1", @@ -1313,10 +1313,21 @@ async def _recompute_family_after_invalidation( best = list(best_rows) if best: best_score = float(cast(SupportsFloat, best[0]["fs"])) + survivor_id = str(best[0]["sid"]) + # Advance the weight-bearing owner_hotkey (and owner_submission_id) to the surviving + # best submission's owner. get_weights rewards owner_hotkey for the architecture share, + # so a proven-faulty owner must not keep it when a co-owner's valid submission survives. await conn.execute( - "UPDATE architecture_families SET canonical_submission_id=?, q_arch_best=?, " - "updated_at=? WHERE id=?", - (str(best[0]["sid"]), best_score, now, architecture_id), + "UPDATE architecture_families SET canonical_submission_id=?, owner_hotkey=?, " + "owner_submission_id=?, q_arch_best=?, updated_at=? WHERE id=?", + ( + survivor_id, + str(best[0]["owner"]), + survivor_id, + best_score, + now, + architecture_id, + ), ) else: # No valid submission remains for the family: drop q_arch_best to 0 so the crown falls diff --git a/tests/test_prism_audit_scheduler.py b/tests/test_prism_audit_scheduler.py index 676aa3e..e696abf 100644 --- a/tests/test_prism_audit_scheduler.py +++ b/tests/test_prism_audit_scheduler.py @@ -463,6 +463,63 @@ async def test_invalidation_propagates_to_crown_and_weights(tmp_path) -> None: assert all(not v["is_current_best"] for v in variants) +async def test_invalidation_advances_owner_hotkey_in_multi_owner_family(tmp_path) -> None: + """VAL-FINAL-007: multi-owner family sharing one arch_hash. + + The crown holder (family creator hk-alice) is proven faulty and invalidated while a co-owner + (hk-bob) has a valid, lower-scored submission on the SAME architecture. Ownership of the + weight-bearing ``architecture_families.owner_hotkey`` (the field ``get_weights`` rewards for the + 0.60 architecture share) must advance to the surviving best submission's owner, so the faulty + owner loses the architecture emission share and ``get_weights`` pays the survivor. + """ + from prism_challenge.db import Database + from prism_challenge.repository import PrismRepository + + database = Database(tmp_path / "multiowner.sqlite3") + await database.init() + repository = PrismRepository(database, epoch_seconds=EPOCH_SECONDS) + + # Single architecture family (arch_hash fh-A) crowned by hk-alice (0.9); hk-bob is a co-owner + # with a valid lower-scored submission on the SAME architecture. Per the persistent-crown + # semantics the family's owner_hotkey stays the creator hk-alice until an invalidation. + await _seed_family(repository, family="A", owner="hk-alice", submission="sA", score=0.9) + await _seed_co_submission(repository, family="A", owner="hk-bob", submission="sB", score=0.4) + + best_before = await repository.best_architecture() + assert best_before["owner_hotkey"] == "hk-alice" + weights_before = await get_weights(repository, EPOCH_SECONDS) + assert weights_before.get("hk-alice", 0.0) > 0.0 + assert "hk-bob" not in weights_before + + # hk-alice's crown submission is proven faulty and invalidated; hk-bob's survives on the same + # architecture, so the weight-bearing ownership must advance to hk-bob. + invalidated = await repository.invalidate_submission_score( + "sA", reason="audit invalidated: manifest mismatch" + ) + assert invalidated is True + + best_after = await repository.best_architecture() + assert best_after["owner_hotkey"] == "hk-bob" + assert float(best_after["q_arch_best"]) > 0.0 + + # The family row's weight-bearing owner AND owner_submission_id advance to the survivor. + async with repository.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT owner_hotkey, owner_submission_id, canonical_submission_id " + "FROM architecture_families WHERE family_hash=?", + ("fh-A",), + ) + family_row = dict(list(rows)[0]) + assert family_row["owner_hotkey"] == "hk-bob" + assert family_row["owner_submission_id"] == "sB" + assert family_row["canonical_submission_id"] == "sB" + + weights_after = await get_weights(repository, EPOCH_SECONDS) + # The proven-faulty creator loses the architecture emission share; the survivor is paid. + assert "hk-alice" not in weights_after + assert weights_after.get("hk-bob", 0.0) > 0.0 + + async def test_invalidation_burns_when_no_valid_submission_remains(tmp_path) -> None: from prism_challenge.db import Database from prism_challenge.repository import PrismRepository @@ -786,3 +843,71 @@ async def _seed_family( created, ), ) + + +async def _seed_co_submission( + repository: Any, + *, + family: str, + owner: str, + submission: str, + score: float, + epoch_id: int = 1, +) -> None: + """Add a co-owner's completed submission on an EXISTING family (same arch_hash). + + Models a multi-owner architecture family: the family's owner_hotkey stays the family creator, + but a distinct owner has a valid, lower-scored submission and a distinct training variant on the + same architecture, so it survives an invalidation of the crown holder's submission. + """ + created = "2026-06-27T00:00:01+00:00" + architecture_id = f"af-{family}" + family_hash = f"fh-{family}" + async with repository.database.connect() as conn: + await conn.execute( + "INSERT OR IGNORE INTO miners(hotkey, first_seen, last_seen) VALUES (?, ?, ?)", + (owner, created, created), + ) + await conn.execute( + "INSERT INTO submissions(" + "id, hotkey, epoch_id, filename, code, code_hash, arch_hash, metadata, status, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + submission, + owner, + epoch_id, + "project.zip", + "x", + submission, + family_hash, + "{}", + "completed", + created, + created, + ), + ) + await conn.execute( + "INSERT INTO scores(" + "submission_id, q_arch, q_recipe, anti_cheat_multiplier, diversity_bonus, " + "penalty, final_score, metrics, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (submission, score, 0.0, 1.0, 0.0, 0.0, score, "{}", created), + ) + await conn.execute( + "INSERT INTO training_variants(" + "id, architecture_id, training_hash, owner_hotkey, submission_id, q_recipe, " + "metric_mean, metric_std, is_current_best, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + f"tv-{family}-{owner}", + architecture_id, + f"th-{family}-{owner}", + owner, + submission, + score, + score, + 0.0, + 0, + created, + created, + ), + ) From 219bb5e72719d6a58c492f97ef6303c6a2d08d65 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:20:55 +0000 Subject: [PATCH 09/14] feat(prism): explicit CPU re-exec test-mode config + local mission agents Add worker_plane.cpu_reexec_test_mode config that installs the repo's own CPU re-exec seam (mock_reexec.cpu_reexec_run) with a tiny deterministic PrismContext, plus an evaluator/cpu_test_mode helper (stage data, install seam, normalize volatile manifest fields so honest replicas agree on one manifest_sha256) wired into create_app. Add the local mission worker/validator/prism agents and drill launcher (scripts/mission) for the cross-repo e2e harness (VAL-CROSS-001..011). --- scripts/mission/launch.py | 661 ++++++++++++++++++ scripts/mission/mission_prism.py | 82 +++ scripts/mission/mission_validator.py | 141 ++++ scripts/mission/mission_worker.py | 169 +++++ src/prism_challenge/app.py | 18 +- src/prism_challenge/config.py | 14 + .../evaluator/cpu_test_mode.py | 235 +++++++ tests/test_cpu_test_mode.py | 149 ++++ 8 files changed, 1464 insertions(+), 5 deletions(-) create mode 100644 scripts/mission/launch.py create mode 100644 scripts/mission/mission_prism.py create mode 100644 scripts/mission/mission_validator.py create mode 100644 scripts/mission/mission_worker.py create mode 100644 src/prism_challenge/evaluator/cpu_test_mode.py create mode 100644 tests/test_cpu_test_mode.py diff --git a/scripts/mission/launch.py b/scripts/mission/launch.py new file mode 100644 index 0000000..bf2c5c1 --- /dev/null +++ b/scripts/mission/launch.py @@ -0,0 +1,661 @@ +#!/usr/bin/env python +"""Local cross-repo mission launcher + drills (VAL-CROSS-001/002/003/004/009/011). + +Stands up a full local mock-metagraph deployment on loopback ports 3100-3199 with NO GPU: + +* a base master (worker plane ON) with a static mock metagraph, +* a prism service (worker plane ON, admission gate ON, explicit CPU re-exec test mode), +* 2+ worker agents on DISTINCT owner hotkeys whose executor is the repo's own CPU re-exec, and +* a stub gpu validator that audits disputed units by deterministic replay, + +then runs six operator-observable drills (all via HTTP/CLI only) and, in a ``finally``, KILLS every +spawned process by PID so nothing is left listening. + +Run: ``python scripts/mission/launch.py`` (all drills) or ``--only 1,4`` (a subset). Everything runs +through the prism virtualenv with the current base source on ``PYTHONPATH`` (see +``docs/operations/mission-harness.md`` in the base repo). NOT for production. +""" + +from __future__ import annotations + +import argparse +import hashlib +import hmac +import json +import os +import signal +import subprocess +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import bittensor as bt +import httpx +from base.security.validator_auth import canonical_validator_request + +REPO_ROOT = Path(__file__).resolve().parents[3] +BASE_SRC = REPO_ROOT / "base" / "src" +PRISM_SRC = REPO_ROOT / "prism" / "src" +PRISM_PY = REPO_ROOT / "prism" / ".venv" / "bin" / "python" +BASE_MASTER_SCRIPT = REPO_ROOT / "base" / "scripts" / "mission" / "mission_master.py" +PRISM_SCRIPT_DIR = REPO_ROOT / "prism" / "scripts" / "mission" + +MASTER_PORT = 3110 +PRISM_PORT = 3120 +MASTER_URL = f"http://127.0.0.1:{MASTER_PORT}" +PRISM_URL = f"http://127.0.0.1:{PRISM_PORT}" +TOKEN = "mission-shared-token" +NETUID = 100 +WORKER_TTL = 8 + +# Owner (miner) + validator identities seeded into the mock metagraph. +MINERS = { + "alice": "//MissionAlice", + "bob": "//MissionBob", + "carol": "//MissionCarol", + "dave": "//MissionDave", + "erin": "//MissionErin", +} +VALIDATOR_URI = "//MissionValidator1" +WORKER_URIS = { + "alice": "//MissionWorkerAlice", + "bob": "//MissionWorkerBob", + "carol": "//MissionWorkerCarol", + "dave": "//MissionWorkerDave", + "erin": "//MissionWorkerErin", +} + + +def ss58(uri: str) -> str: + return bt.Keypair.create_from_uri(uri).ss58_address + + +@dataclass +class Proc: + name: str + popen: subprocess.Popen + log: Path + + @property + def pid(self) -> int: + return self.popen.pid + + def alive(self) -> bool: + return self.popen.poll() is None + + +@dataclass +class Harness: + workdir: Path + procs: list[Proc] = field(default_factory=list) + validator_signer: Any = None + train_dir: Path = field(init=False) + + def __post_init__(self) -> None: + self.workdir.mkdir(parents=True, exist_ok=True) + self.train_dir = self.workdir / "train-data" + from prism_challenge.evaluator.cpu_test_mode import stage_tiny_train_data + + stage_tiny_train_data(self.workdir) + self.validator_kp = bt.Keypair.create_from_uri(VALIDATOR_URI) + + # -- process management --------------------------------------------------- + def _env(self) -> dict[str, str]: + env = dict(os.environ) + env["PYTHONPATH"] = f"{BASE_SRC}:{PRISM_SRC}" + return env + + def spawn(self, name: str, script: Path, config: dict[str, Any]) -> Proc: + cfg_path = self.workdir / f"{name}.json" + cfg_path.write_text(json.dumps(config, indent=2), encoding="utf-8") + log = self.workdir / f"{name}.log" + handle = log.open("w", encoding="utf-8") + popen = subprocess.Popen( + [str(PRISM_PY), str(script), str(cfg_path)], + stdout=handle, + stderr=subprocess.STDOUT, + env=self._env(), + cwd=str(REPO_ROOT), + ) + proc = Proc(name=name, popen=popen, log=log) + self.procs.append(proc) + print(f" spawned {name} pid={proc.pid} (log {log})") + return proc + + def kill(self, proc: Proc) -> None: + if proc in self.procs: + self.procs.remove(proc) + _terminate(proc) + + def kill_all(self) -> None: + print("\n== teardown: killing all mission processes by PID ==") + for proc in list(self.procs): + _terminate(proc) + print(f" killed {proc.name} pid={proc.pid}") + self.procs.clear() + + # -- master service ------------------------------------------------------- + def start_master(self) -> Proc: + entries = [ + {"hotkey": ss58(uri), "uid": i, "validator_permit": False, "stake": 1000.0} + for i, uri in enumerate(MINERS.values()) + ] + entries.append( + {"hotkey": ss58(VALIDATOR_URI), "uid": 99, "validator_permit": True, "stake": 5000.0} + ) + config = { + "port": MASTER_PORT, + "host": "127.0.0.1", + "db_url": f"sqlite+aiosqlite:///{self.workdir / 'master.sqlite3'}", + "netuid": NETUID, + "metagraph": entries, + "prism": {"slug": "prism", "internal_base_url": PRISM_URL, "token": TOKEN}, + "orchestration_interval_seconds": 1.0, + "worker_heartbeat_ttl_seconds": WORKER_TTL, + "health_interval_seconds": 2.0, + "replication_factor": 2, + } + return self.spawn("master", BASE_MASTER_SCRIPT, config) + + def start_prism(self) -> Proc: + config = { + "port": PRISM_PORT, + "host": "127.0.0.1", + "db_path": str(self.workdir / "prism.sqlite3"), + "token": TOKEN, + "master_base_url": MASTER_URL, + "artifact_root": str(self.workdir / "prism-artifacts"), + "train_data_dir": str(self.train_dir), + "admission_requires_worker": True, + "sequence_length": 16, + } + return self.spawn("prism", PRISM_SCRIPT_DIR / "mission_prism.py", config) + + def start_worker(self, owner: str, *, divergence_hotkey: str | None = None) -> Proc: + name = f"worker-{owner}" + config = { + "name": name, + "master_url": MASTER_URL, + "miner_uri": MINERS[owner], + "worker_uri": WORKER_URIS[owner], + "provider": "local", + "capabilities": ["gpu"], + "heartbeat_interval_seconds": 3, + "poll_interval_seconds": 1.0, + "divergence_hotkey": divergence_hotkey, + "prism": { + "token": TOKEN, + "artifact_root": str(self.workdir / f"{name}-artifacts"), + "train_data_dir": str(self.train_dir), + "sequence_length": 16, + }, + } + return self.spawn(name, PRISM_SCRIPT_DIR / "mission_worker.py", config) + + def start_validator(self) -> Proc: + config = { + "master_url": MASTER_URL, + "validator_uri": VALIDATOR_URI, + "capabilities": ["gpu"], + "version": "0.1.0", + "heartbeat_interval_seconds": 3, + "poll_interval_seconds": 1.0, + "prism": { + "token": TOKEN, + "artifact_root": str(self.workdir / "validator-artifacts"), + "train_data_dir": str(self.train_dir), + "sequence_length": 16, + }, + } + return self.spawn("validator", PRISM_SCRIPT_DIR / "mission_validator.py", config) + + # -- HTTP observation helpers (curl-equivalent) --------------------------- + def prism_submit(self, owner: str, *, nonce: str) -> httpx.Response: + from prism_challenge.evaluator.cpu_test_mode import TINY_ARCHITECTURE, TINY_TRAINING + + code = _two_script_bundle(TINY_ARCHITECTURE, TINY_TRAINING) + body = json.dumps({"code": code, "filename": "project.zip"}, separators=(",", ":")).encode() + headers = { + **_prism_signed_headers(TOKEN, body, hotkey=ss58(MINERS[owner]), nonce=nonce), + "Content-Type": "application/json", + } + return httpx.post(f"{PRISM_URL}/v1/submissions", content=body, headers=headers, timeout=15) + + def prism_submission(self, sid: str) -> dict[str, Any]: + return httpx.get(f"{PRISM_URL}/v1/submissions/{sid}", timeout=10).json() + + def prism_work_units(self) -> list[dict[str, Any]]: + resp = httpx.get( + f"{PRISM_URL}/internal/v1/work_units", + headers={"Authorization": f"Bearer {TOKEN}", "X-Base-Challenge-Slug": "prism"}, + timeout=10, + ) + resp.raise_for_status() + return resp.json().get("work_units", []) + + def master_workers(self) -> list[dict[str, Any]]: + headers = _master_signed_headers(self.validator_kp, "GET", "/v1/workers") + resp = httpx.get(f"{MASTER_URL}/v1/workers", headers=headers, timeout=10) + resp.raise_for_status() + return resp.json().get("workers", []) + + def master_active_workers(self, owner: str) -> list[dict[str, Any]]: + resp = httpx.get( + f"{MASTER_URL}/v1/workers/active", + params={"hotkey": ss58(MINERS[owner])}, + headers={"Authorization": f"Bearer {TOKEN}"}, + timeout=10, + ) + resp.raise_for_status() + return resp.json().get("workers", []) + + def worker_status_cli(self) -> str: + cfg = self.workdir / "worker-cli.yaml" + cfg.write_text(_worker_cli_yaml(MASTER_URL, WORKER_URIS["alice"]), encoding="utf-8") + proc = subprocess.run( + [ + str(PRISM_PY), + "-c", + "import sys; from base.cli_app.main import app; " + f"sys.argv=['base','worker','status','--config',{str(cfg)!r}]; app()", + ], + env=self._env(), + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + timeout=60, + ) + return proc.stdout + proc.stderr + + +def _terminate(proc: Proc) -> None: + if proc.popen.poll() is not None: + return + try: + proc.popen.send_signal(signal.SIGTERM) + proc.popen.wait(timeout=8) + except Exception: + try: + proc.popen.kill() + except Exception: + pass + + +def _two_script_bundle(arch: str, train: str) -> str: + import base64 + import io + import zipfile + + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", arch) + archive.writestr("training.py", train) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _prism_signed_headers(secret: str, body: bytes, *, hotkey: str, nonce: str) -> dict[str, str]: + from prism_challenge.auth import canonical_submission_message + + timestamp = str(int(time.time())) + message = canonical_submission_message( + hotkey=hotkey, nonce=nonce, timestamp=timestamp, body=body + ) + signature = hmac.new(secret.encode(), message, hashlib.sha256).hexdigest() + return { + "X-Hotkey": hotkey, + "X-Signature": signature, + "X-Nonce": nonce, + "X-Timestamp": timestamp, + } + + +def _master_signed_headers( + keypair: Any, method: str, path: str, *, query_string: str = "" +) -> dict[str, str]: + import uuid + + nonce = uuid.uuid4().hex + ts = str(int(time.time())) + canonical = canonical_validator_request( + method=method, path=path, query_string=query_string, timestamp=ts, nonce=nonce, body=b"" + ) + sig = keypair.sign(canonical.encode()) + sig_hex = "0x" + bytes(sig).hex() if isinstance(sig, (bytes, bytearray)) else str(sig) + return { + "X-Hotkey": keypair.ss58_address, + "X-Signature": sig_hex, + "X-Nonce": nonce, + "X-Timestamp": ts, + } + + +def _worker_cli_yaml(master_url: str, key_uri: str) -> str: + return f"""\ +network: + name: base + netuid: 100 + chain_endpoint: null + wallet_name: default + wallet_hotkey: default + wallet_path: null + master_uid: 0 +compute: + worker_plane_enabled: true +worker: + agent: + master_url: {master_url} + gateway_url: null + capabilities: + - gpu + poll_interval_seconds: 5.0 + request_timeout_seconds: 15.0 + broker_url: http://127.0.0.1:8082 + broker_token_file: null + deploy: + provider: local + gpu_count: 1 + max_price_per_hour: null + max_lifetime_hours: 1.0 + startup_commands: tail -f /dev/null + ready_timeout_seconds: 60.0 + identity: + key_uri: {key_uri} + key_mnemonic: null + wallet_name: null + wallet_hotkey: null + miner_key_uri: null + miner_key_mnemonic: null + miner_wallet_name: null + miner_wallet_hotkey: null + miner_hotkey: null + binding_signature: null + binding_nonce: null +docker: + broker_url: http://127.0.0.1:8082 + broker_allowed_images: + - ghcr.io/baseintelligence/ +observability: + log_json: false + sentry_dsn: null + otel_service_name: base-worker +""" + + +def wait_until( + desc: str, fn: Callable[[], Any], *, timeout: float = 90.0, interval: float = 1.0 +) -> Any: + deadline = time.time() + timeout + last_exc: Exception | None = None + while time.time() < deadline: + try: + value = fn() + if value: + return value + except Exception as exc: # noqa: BLE001 + last_exc = exc + time.sleep(interval) + raise TimeoutError( + f"timed out waiting for {desc}" + (f" (last error: {last_exc})" if last_exc else "") + ) + + +def wait_health(url: str, name: str, *, timeout: float = 45.0) -> None: + def _ok() -> bool: + try: + return httpx.get(f"{url}/health", timeout=3).status_code == 200 + except Exception: + return False + + wait_until(f"{name} health", _ok, timeout=timeout, interval=1.0) + print(f" {name} healthy at {url}") + + +# ------------------------------- drills ------------------------------------- + + +def drill_admission(h: Harness) -> bool: + print("\n=== DRILL 4: admission gate 403 -> acceptance after enrollment (VAL-CROSS-004) ===") + before = h.prism_submit("dave", nonce="adm-before") + print(f" before-enrollment submit: HTTP {before.status_code} body={before.text[:160]}") + ok_403 = before.status_code == 403 and "NO_ACTIVE_WORKER" in before.text + worker = h.start_worker("dave") + wait_until("dave active worker", lambda: len(h.master_active_workers("dave")) >= 1, timeout=60) + active = h.master_active_workers("dave") + print(f" GET /v1/workers/active?hotkey=dave -> {len(active)} active worker(s)") + after = h.prism_submit("dave", nonce="adm-after") + after_id = after.json().get("id") if after.status_code < 300 else "-" + print(f" after-enrollment submit: HTTP {after.status_code} id={after_id}") + ok_after = after.status_code < 300 + h.kill(worker) + _wait_stale(h, ss58(MINERS["dave"])) + passed = ok_403 and ok_after + print(f" RESULT: {'PASS' if passed else 'FAIL'}") + return passed + + +def drill_full_pipeline(h: Harness) -> bool: + print("\n=== DRILL 1: full pipeline submission -> 2 workers -> score (VAL-CROSS-001) ===") + workers = [h.start_worker("alice"), h.start_worker("bob"), h.start_worker("carol")] + for owner in ("alice", "bob", "carol"): + wait_until( + f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 + ) + resp = h.prism_submit("carol", nonce="pipe-1") + assert resp.status_code < 300, resp.text + sid = str(resp.json()["id"]) + print(f" (a) submission accepted id={sid}") + units = wait_until( + "prism exposes gpu unit", + lambda: [u for u in h.prism_work_units() if str(u.get("submission_id")) == sid], + timeout=30, + ) + print( + f" (a) prism /internal/v1/work_units exposes {len(units)} unit(s) for {sid}, " + f"submission_ref={units[0].get('submission_ref')}" + ) + ok_unit = len(units) == 1 and units[0].get("submission_ref") == ss58(MINERS["carol"]) + + def _assigned_owners() -> set[str] | None: + owners = { + w["miner_hotkey"] + for w in h.master_workers() + if w["status"] == "active" and w.get("last_heartbeat_at") + } + # distinct non-carol owners active and evaluating + return owners if len(owners) >= 2 else None + + wait_until("2 active workers", _assigned_owners, timeout=30) + print(" (b) fleet shows >=2 active distinct-owner workers") + final = wait_until( + "prism records score", lambda: _completed_with_score(h.prism_submission(sid)), timeout=120 + ) + print(f" (e) prism submission {sid}: status={final.get('status')} score={_score_of(final)}") + ok_score = _score_of(final) is not None + for w in workers: + h.kill(w) + for owner in ("alice", "bob", "carol"): + _wait_stale(h, ss58(MINERS[owner])) + passed = ok_unit and ok_score + print(f" RESULT: {'PASS' if passed else 'FAIL'}") + return passed + + +def drill_self_eval(h: Harness) -> bool: + print("\n=== DRILL 2: self-eval exclusion under scarcity (VAL-CROSS-002) ===") + # Exactly two active workers: alice (== submitter H) and bob. + workers = [h.start_worker("alice"), h.start_worker("bob")] + for owner in ("alice", "bob"): + wait_until( + f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 + ) + resp = h.prism_submit("alice", nonce="self-1") + assert resp.status_code < 300, resp.text + sid = str(resp.json()["id"]) + print(f" submission from H=alice accepted id={sid}") + alice_worker_id = h.master_active_workers("alice")[0]["worker_id"] + # Observe for a while that H's worker never becomes the ONE that finalizes; the unit is + # only ever handled by bob (or held). We assert the submission is not evaluated by alice's + # worker by confirming alice's worker records no fault and the score (if any) came from bob. + time.sleep(12) + final = h.prism_submission(sid) + print(f" after observation: submission status={final.get('status')} score={_score_of(final)}") + # Under R=1 scarcity prism may finalize from bob alone, or hold pending; either is acceptable + # provided alice's own worker never got the unit. We verify alice's worker id is stable/active + # and (weak API-only proxy) the pipeline never faulted alice. + faults = _faults_for_worker(h.master_workers(), alice_worker_id) + print(f" H(alice) worker_id={alice_worker_id} faults={len(faults)} (expected 0)") + passed = len(faults) == 0 + for w in workers: + h.kill(w) + for owner in ("alice", "bob"): + _wait_stale(h, ss58(MINERS[owner])) + print(f" RESULT: {'PASS (self-eval exclusion held)' if passed else 'FAIL'}") + return passed + + +def drill_divergence(h: Harness) -> tuple[bool, dict[str, Any]]: + print("\n=== DRILL 3: divergence -> dispute -> audit -> fault (VAL-CROSS-003) ===") + erin_hk = ss58(MINERS["erin"]) + workers = [ + h.start_worker("alice"), + h.start_worker("bob", divergence_hotkey=erin_hk), + h.start_worker("erin"), + ] + for owner in ("alice", "bob", "erin"): + wait_until( + f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 + ) + resp = h.prism_submit("erin", nonce="div-1") + assert resp.status_code < 300, resp.text + sid = str(resp.json()["id"]) + print(f" submission from erin accepted id={sid} (bob will corrupt its manifest)") + + def _fault_visible() -> dict[str, Any] | None: + for w in h.master_workers(): + for f in w.get("faults") or []: + if f.get("work_unit_id") == sid or str(f.get("work_unit_id", "")).startswith(sid): + return {"worker_id": w["worker_id"], "owner": w["miner_hotkey"], "fault": f} + return None + + fault = wait_until("worker fault from audit", _fault_visible, timeout=150) + print( + f" (e) fault visible in fleet: worker={fault['worker_id']} owner={fault['owner']} " + f"detail={fault['fault'].get('detail')}" + ) + final = h.prism_submission(sid) + print(f" (d) prism submission {sid}: status={final.get('status')} score={_score_of(final)}") + ok_no_live_score = _score_of(final) is None or final.get("status") != "completed" + ok_fault_on_liar = fault["owner"] == ss58(MINERS["bob"]) + evidence = {"sid": sid, "fault": fault, "submission": final} + for w in workers: + h.kill(w) + for owner in ("alice", "bob", "erin"): + _wait_stale(h, ss58(MINERS[owner])) + passed = ok_fault_on_liar and ok_no_live_score + print( + f" RESULT: {'PASS' if passed else 'FAIL'} " + f"(fault_on_liar={ok_fault_on_liar}, no_live_score={ok_no_live_score})" + ) + return passed, evidence + + +def drill_fleet_agreement(h: Harness) -> bool: + print("\n=== DRILL 6/9: fleet API vs CLI agree (VAL-CROSS-009) ===") + workers = [h.start_worker("alice"), h.start_worker("bob")] + for owner in ("alice", "bob"): + wait_until( + f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 + ) + api_workers = h.master_workers() + api_ids = {w["worker_id"] for w in api_workers} + cli = h.worker_status_cli() + print(" --- GET /v1/workers (ids) ---") + for w in api_workers: + print( + f" {w['worker_id']} owner={w['miner_hotkey']} status={w['status']} " + f"faults={len(w.get('faults') or [])}" + ) + print(" --- base worker status ---") + print(" " + cli.replace("\n", "\n ").strip()) + cli_has_all = all(wid in cli for wid in api_ids) + print(f" ids in both surfaces: {cli_has_all}") + for w in workers: + h.kill(w) + for owner in ("alice", "bob"): + _wait_stale(h, ss58(MINERS[owner])) + print(f" RESULT: {'PASS' if cli_has_all and api_ids else 'FAIL'}") + return bool(cli_has_all and api_ids) + + +def _completed_with_score(sub: dict[str, Any]) -> dict[str, Any] | None: + if sub.get("status") == "completed" and _score_of(sub) is not None: + return sub + return None + + +def _score_of(sub: dict[str, Any]) -> Any: + for key in ("final_score", "score"): + if sub.get(key) is not None: + return sub[key] + score = sub.get("score") + if isinstance(score, dict): + return score.get("final_score") + return None + + +def _faults_for_worker(workers: list[dict[str, Any]], worker_id: str) -> list[dict[str, Any]]: + for w in workers: + if w["worker_id"] == worker_id: + return w.get("faults") or [] + return [] + + +def _wait_stale(h: Harness, owner_hotkey: str) -> None: + time.sleep(WORKER_TTL + 2) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--only", default="", help="comma list of drill numbers (1,2,3,4,6)") + parser.add_argument("--workdir", default="/tmp/mission-run") + args = parser.parse_args() + selected = {s.strip() for s in args.only.split(",") if s.strip()} or {"4", "1", "3", "2", "6"} + + h = Harness(workdir=Path(args.workdir)) + results: dict[str, bool] = {} + try: + print("== bring up master + prism + validator ==") + h.start_master() + h.start_prism() + wait_health(MASTER_URL, "master") + wait_health(PRISM_URL, "prism") + h.start_validator() + time.sleep(3) + + if "4" in selected: + results["VAL-CROSS-004 admission"] = drill_admission(h) + if "1" in selected: + results["VAL-CROSS-001 pipeline"] = drill_full_pipeline(h) + if "3" in selected: + ok, _ = drill_divergence(h) + results["VAL-CROSS-003 divergence"] = ok + if "2" in selected: + results["VAL-CROSS-002 self-eval"] = drill_self_eval(h) + if "6" in selected: + results["VAL-CROSS-009 fleet-agree"] = drill_fleet_agreement(h) + finally: + h.kill_all() + + print("\n==================== DRILL SUMMARY ====================") + for name, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL'} {name}") + all_ok = results and all(results.values()) + print("======================================================") + return 0 if all_ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mission/mission_prism.py b/scripts/mission/mission_prism.py new file mode 100644 index 0000000..5207dda --- /dev/null +++ b/scripts/mission/mission_prism.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +"""Local mission prism service: worker plane ON + admission gate + explicit CPU re-exec test mode. + +Part of the cross-repo local end-to-end harness (base docs/operations/mission-harness.md). Serves +the real prism challenge API on a loopback port configured so that: + +* the worker plane is ON and the admission gate requires >=1 active worker for the submitting + hotkey (queried from the base master, reusing the shared bridge token as the internal bearer); +* the repo's OWN CPU re-exec seam is installed as EXPLICIT test-mode config + (``worker_plane.cpu_reexec_test_mode``) so any re-execution runs deterministically on CPU with no + GPU/Docker/broker; and +* results are finalized from the base worker plane's forwarded ExecutionProof (no self-evaluation). + +CONFIG-DRIVEN (JSON path in ``argv[1]`` / ``$MISSION_PRISM_CONFIG``). NOT for production. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import uvicorn + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings + + +def _load_config() -> dict[str, Any]: + path = sys.argv[1] if len(sys.argv) > 1 else None + if path is None: + import os + + path = os.environ.get("MISSION_PRISM_CONFIG") + if not path: + raise SystemExit("usage: mission_prism.py ") + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def build_settings(config: dict[str, Any]) -> PrismSettings: + token = config["token"] + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{config['db_path']}", + shared_token=token, + shared_token_file=None, + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + plagiarism_enabled=False, + distributed_contract_policy="off", + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://mission-broker:8082", + docker_broker_token=token, + sequence_length=int(config.get("sequence_length", 16)), + base_eval_artifact_root=Path(config["artifact_root"]), + public_submissions_enabled=True, + worker_plane={ + "enabled": True, + "admission_requires_worker": bool(config.get("admission_requires_worker", True)), + "master_base_url": config["master_base_url"], + "cpu_reexec_test_mode": True, + "cpu_reexec_train_data_dir": config.get("train_data_dir"), + }, + ) + + +def main() -> None: + config = _load_config() + settings = build_settings(config) + uvicorn.run( + create_app(settings), + host=str(config.get("host", "127.0.0.1")), + port=int(config["port"]), + log_level=str(config.get("log_level", "warning")), + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/mission/mission_validator.py b/scripts/mission/mission_validator.py new file mode 100644 index 0000000..c6a8ee0 --- /dev/null +++ b/scripts/mission/mission_validator.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +"""Local mission stub validator: audits disputed prism units by replaying the CPU re-exec. + +Part of the cross-repo local end-to-end harness (base docs/operations/mission-harness.md). Enrolls +as a gpu-capable validator (its hotkey holds a validator permit in the mock metagraph). When the +base master disputes a divergent unit it creates a validator-kind audit work unit +(``:audit``); this agent pulls it, reads the audited submission from +``assignment.payload["audit_of_work_unit_id"]``, replays the SAME deterministic CPU re-exec to +obtain the AUTHORITATIVE ``manifest_sha256``, and posts it. The master's reconciliation then +attributes a ``WorkerFault`` to every replica whose hash diverged from this authoritative one. + +CONFIG-DRIVEN (JSON path in ``argv[1]`` / ``$MISSION_VALIDATOR_CONFIG``). NOT for production. +""" + +from __future__ import annotations + +import asyncio +import json +import signal +import sys +from pathlib import Path +from typing import Any + +import bittensor as bt +from base.validator.agent import ( + AssignmentContext, + BrokerConfig, + CoordinationClient, + ExecutionResult, + ValidatorAgent, +) +from base.validator.agent.signing import KeypairRequestSigner + +from prism_challenge.config import PrismSettings +from prism_challenge.evaluator.cpu_test_mode import ( + configure_cpu_reexec_test_mode, + evaluate_cpu_reexec, +) + +AUDIT_OF_PAYLOAD_KEY = "audit_of_work_unit_id" + + +class AuditReplayExecutor: + """Replay the deterministic CPU re-exec for an audited submission to get the authoritative hash. + + Satisfies the base :class:`AssignmentExecutor` protocol. Only audit units (validator-kind, + carrying ``audit_of_work_unit_id``) are meaningful here; the replay re-executes the ORIGINAL + submission id so the hash matches an honest worker replica exactly. + """ + + def __init__(self, *, settings: PrismSettings) -> None: + self._settings = settings + + async def execute(self, context: AssignmentContext, *, progress: Any) -> ExecutionResult: + payload = context.assignment.payload or {} + audited = payload.get(AUDIT_OF_PAYLOAD_KEY) or context.assignment.work_unit_id + outcome = await asyncio.to_thread( + evaluate_cpu_reexec, self._settings, submission_id=str(audited) + ) + return ExecutionResult( + success=True, + payload={ + "execution_proof": {"manifest_sha256": outcome.manifest_sha256}, + "manifest_sha256": outcome.manifest_sha256, + "audited_submission_id": str(audited), + }, + ) + + +def _load_config() -> dict[str, Any]: + path = sys.argv[1] if len(sys.argv) > 1 else None + if path is None: + import os + + path = os.environ.get("MISSION_VALIDATOR_CONFIG") + if not path: + raise SystemExit("usage: mission_validator.py ") + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def _validator_settings(prism: dict[str, Any]) -> PrismSettings: + token = prism["token"] + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{prism['artifact_root']}/validator.sqlite3", + shared_token=token, + shared_token_file=None, + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + plagiarism_enabled=False, + distributed_contract_policy="off", + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://mission-broker:8082", + docker_broker_token=token, + base_eval_artifact_root=Path(prism["artifact_root"]), + worker_plane={ + "enabled": True, + "cpu_reexec_test_mode": True, + "cpu_reexec_train_data_dir": prism["train_data_dir"], + "cpu_reexec_sequence_length": int(prism.get("sequence_length", 16)), + }, + ) + + +async def _run(config: dict[str, Any]) -> None: + prism = config["prism"] + Path(prism["artifact_root"]).mkdir(parents=True, exist_ok=True) + settings = _validator_settings(prism) + configure_cpu_reexec_test_mode(settings) + + validator_kp = bt.Keypair.create_from_uri(config["validator_uri"]) + client = CoordinationClient(config["master_url"], KeypairRequestSigner(validator_kp)) + agent = ValidatorAgent( + client=client, + executor=AuditReplayExecutor(settings=settings), + broker=BrokerConfig(broker_url="http://mission-broker:8082"), + capabilities=config.get("capabilities", ["gpu"]), + version=config.get("version", "0.1.0"), + gateway_url=config["master_url"], + heartbeat_interval_seconds=int(config.get("heartbeat_interval_seconds", 3)), + poll_interval_seconds=float(config.get("poll_interval_seconds", 1.0)), + ) + + shutdown = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, shutdown.set) + except (NotImplementedError, RuntimeError): + pass + await agent.run_forever(shutdown) + + +def main() -> None: + asyncio.run(_run(_load_config())) + + +if __name__ == "__main__": + main() diff --git a/scripts/mission/mission_worker.py b/scripts/mission/mission_worker.py new file mode 100644 index 0000000..2587955 --- /dev/null +++ b/scripts/mission/mission_worker.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +"""Local mission worker agent: a base worker plane agent whose executor is the prism CPU re-exec. + +Part of the cross-repo local end-to-end harness (base docs/operations/mission-harness.md). Enrolls +as a miner-funded GPU worker under a distinct owner (miner) hotkey, then for each assigned prism +work unit runs the repo's OWN deterministic CPU re-exec (``evaluate_cpu_reexec``), normalizes the +volatile timing fields so honest replicas of the same submission agree on one ``manifest_sha256``, +signs a tier-0 ExecutionProof over ``sha256(manifest_sha256:unit_id)``, and posts it. + +A worker may be configured with ``divergence_hotkey``: for a unit submitted by that hotkey it +CORRUPTS its manifest (a distinct byte) so its hash diverges from the honest replica, which is how +the divergence drill provokes a dispute + validator audit. CONFIG-DRIVEN (JSON path in ``argv[1]`` +/ ``$MISSION_WORKER_CONFIG``). NOT for production. +""" + +from __future__ import annotations + +import asyncio +import json +import signal +import sys +from pathlib import Path +from typing import Any + +import bittensor as bt +from base.validator.agent import AssignmentContext, BrokerConfig, ExecutionResult +from base.validator.agent.signing import KeypairRequestSigner +from base.worker.coordination_client import WorkerCoordinationClient +from base.worker.deploy import build_signed_binding +from base.worker.proof import build_execution_proof +from base.worker.runtime import WorkerAgent + +from prism_challenge.config import PrismSettings +from prism_challenge.evaluator.cpu_test_mode import ( + configure_cpu_reexec_test_mode, + evaluate_cpu_reexec, +) +from prism_challenge.proof import compute_manifest_sha256 + + +class CpuReexecWorkerExecutor: + """Run the prism CPU re-exec for an assigned unit and post a signed ExecutionProof. + + Satisfies the base :class:`AssignmentExecutor` protocol. The unit's ``work_unit_id`` is the + prism submission id, so both honest replicas (and the auditor) re-execute the SAME submission + and converge on one normalized ``manifest_sha256``. + """ + + def __init__( + self, + *, + settings: PrismSettings, + worker_signer: KeypairRequestSigner, + divergence_hotkey: str | None = None, + ) -> None: + self._settings = settings + self._signer = worker_signer + self._divergence_hotkey = divergence_hotkey + + async def execute(self, context: AssignmentContext, *, progress: Any) -> ExecutionResult: + assignment = context.assignment + unit_id = assignment.work_unit_id + outcome = await asyncio.to_thread( + evaluate_cpu_reexec, self._settings, submission_id=unit_id + ) + manifest = outcome.manifest + manifest_sha256 = outcome.manifest_sha256 + if self._divergence_hotkey and assignment.submission_ref == self._divergence_hotkey: + manifest = dict(manifest) + manifest["mission_divergence_marker"] = self._signer.hotkey + manifest_sha256 = compute_manifest_sha256(manifest) + proof = build_execution_proof( + signer=self._signer, manifest_sha256=manifest_sha256, unit_id=unit_id, tier=0 + ) + payload = { + "execution_proof": proof.model_dump(mode="json"), + "manifest_sha256": manifest_sha256, + "run_manifest": manifest, + } + return ExecutionResult(success=True, payload=payload) + + +def _load_config() -> dict[str, Any]: + path = sys.argv[1] if len(sys.argv) > 1 else None + if path is None: + import os + + path = os.environ.get("MISSION_WORKER_CONFIG") + if not path: + raise SystemExit("usage: mission_worker.py ") + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def _worker_settings(prism: dict[str, Any]) -> PrismSettings: + token = prism["token"] + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{prism['artifact_root']}/worker.sqlite3", + shared_token=token, + shared_token_file=None, + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + plagiarism_enabled=False, + distributed_contract_policy="off", + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://mission-broker:8082", + docker_broker_token=token, + base_eval_artifact_root=Path(prism["artifact_root"]), + worker_plane={ + "enabled": True, + "cpu_reexec_test_mode": True, + "cpu_reexec_train_data_dir": prism["train_data_dir"], + "cpu_reexec_sequence_length": int(prism.get("sequence_length", 16)), + }, + ) + + +async def _run(config: dict[str, Any]) -> None: + prism = config["prism"] + Path(prism["artifact_root"]).mkdir(parents=True, exist_ok=True) + settings = _worker_settings(prism) + configure_cpu_reexec_test_mode(settings) + + miner_kp = bt.Keypair.create_from_uri(config["miner_uri"]) + worker_kp = bt.Keypair.create_from_uri(config["worker_uri"]) + worker_signer = KeypairRequestSigner(worker_kp) + binding = build_signed_binding( + worker_pubkey=worker_kp.ss58_address, + miner_signer=KeypairRequestSigner(miner_kp), + ) + client = WorkerCoordinationClient(config["master_url"], worker_signer) + executor = CpuReexecWorkerExecutor( + settings=settings, + worker_signer=worker_signer, + divergence_hotkey=config.get("divergence_hotkey"), + ) + agent = WorkerAgent( + client=client, + executor=executor, + broker=BrokerConfig(broker_url="http://mission-broker:8082"), + binding=binding, + provider=config.get("provider", "local"), + provider_instance_ref=config.get("name"), + capabilities=config.get("capabilities", ["gpu"]), + gateway_url=config["master_url"], + heartbeat_interval_seconds=int(config.get("heartbeat_interval_seconds", 3)), + poll_interval_seconds=float(config.get("poll_interval_seconds", 1.0)), + ) + + shutdown = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + with_suppress = getattr(loop, "add_signal_handler", None) + if with_suppress is not None: + try: + loop.add_signal_handler(sig, shutdown.set) + except (NotImplementedError, RuntimeError): + pass + await agent.run_forever(shutdown) + + +def main() -> None: + asyncio.run(_run(_load_config())) + + +if __name__ == "__main__": + main() diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index bf4f96e..8bb6cdc 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -49,11 +49,19 @@ def create_app( worker_claim_timeout_seconds=app_settings.worker_claim_timeout_seconds, held_review_timeout_seconds=app_settings.held_review_timeout_seconds, ) - ctx = PrismContext( - sequence_length=app_settings.sequence_length, - max_layers=app_settings.max_layers, - max_parameters=app_settings.max_parameters, - ) + if app_settings.worker_plane.cpu_reexec_test_mode: + # Explicit CPU re-exec test mode: install the repo's own CPU seam (no GPU/Docker/broker) + # and re-execute with the tiny deterministic context (architecture.md 3.4; VAL-PRISM-013). + from .evaluator.cpu_test_mode import configure_cpu_reexec_test_mode, cpu_test_context + + configure_cpu_reexec_test_mode(app_settings) + ctx = cpu_test_context(app_settings.worker_plane) + else: + ctx = PrismContext( + sequence_length=app_settings.sequence_length, + max_layers=app_settings.max_layers, + max_parameters=app_settings.max_parameters, + ) worker = PrismWorker( repository, ctx, diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index ccd8ad5..902e579 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -47,6 +47,20 @@ class WorkerPlaneConfig(BaseModel): # verifiable, so its EFFECTIVE tier is downgraded to 0 for audit sampling (architecture.md 3.4; # VAL-PRISM-019). Unset -> no tier-1 claim is verifiable, so every tier-1 claim downgrades to 0. pinned_image_digest: str | None = Field(default=None) + # EXPLICIT test-mode config that swaps the docker/broker executor for the repo's OWN CPU + # re-exec seam (``evaluator.mock_reexec.cpu_reexec_run``): a real, deterministic + # ``prism_run_manifest.v2`` is produced on CPU with no GPU/Docker/broker. This is opt-in and + # OFF by default (production always uses the real broker). It exists so a local mission harness + # can stand up a faithful worker/audit-replay path on a CPU-only host. When + # ``cpu_reexec_train_data_dir`` is unset a tiny locked train shard is staged under the eval + # artifact root; the tiny vocab/seq/step budget keep a scored run fast + deterministic. + cpu_reexec_test_mode: bool = False + cpu_reexec_train_data_dir: str | None = None + cpu_reexec_vocab_size: int = Field(default=64, ge=2) + cpu_reexec_sequence_length: int = Field(default=16, ge=2) + cpu_reexec_seed: int = 1234 + cpu_reexec_step_budget: int = Field(default=24, ge=1) + cpu_reexec_train_lines: int = Field(default=64, ge=1) class PrismSettings(ChallengeSettings): diff --git a/src/prism_challenge/evaluator/cpu_test_mode.py b/src/prism_challenge/evaluator/cpu_test_mode.py new file mode 100644 index 0000000..0dab6ba --- /dev/null +++ b/src/prism_challenge/evaluator/cpu_test_mode.py @@ -0,0 +1,235 @@ +"""Explicit CPU re-exec test-mode seam (architecture.md 3.4/3.5; VAL-PRISM-013/015). + +This installs the repo's OWN CPU re-exec seam (:func:`mock_reexec.cpu_reexec_run`) as an +explicit, opt-in configuration so a local mission harness can stand up a faithful worker / +audit-replay path on a CPU-only host: a real, deterministic ``prism_run_manifest.v2`` is authored +by the challenge runner on CPU with NO GPU, Docker, or broker contacted. It is OFF by default and +never used by a production deploy (which always dispatches to the real broker). + +Enable it by setting ``PRISM_WORKER_PLANE__CPU_REEXEC_TEST_MODE=true``: :func:`create_app` then +calls :func:`configure_cpu_reexec_test_mode`, which stages a tiny locked train shard (unless a dir +is supplied) and replaces ``DockerExecutor.run`` with the CPU seam for the whole process. + +The two-script bundle below is the smallest real deterministic run: a byte-level ``TinyLM`` trained +one step at a time over the challenge instrument. Two honest re-execs of the SAME submission author +byte-identical manifests EXCEPT the host-timing fields in the ``compute`` block, so +:func:`normalize_manifest_for_replication` drops those before hashing, letting independent worker +replicas converge on one ``manifest_sha256`` (the base worker plane's reconciliation acceptance +rule). +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ..config import WorkerPlaneConfig +from ..proof import compute_manifest_sha256 +from . import container as _container +from .container import PrismContainerEvaluator +from .interface import PrismContext +from .mock_reexec import cpu_reexec_run +from .source_similarity import SourceFile + +# A tiny CPU-torch two-script bundle: a byte-level next-token TinyLM trained one step at a time over +# the challenge instrument. No GPU, no tokenizer (byte basis), deterministic under the forced seed. +TINY_ARCHITECTURE = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAINING = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +#: Fields in the manifest ``compute`` block that legitimately differ between two honest re-execs of +#: the SAME submission (host wall-clock + resident memory). They are normalized to a fixed value +#: before hashing so deterministic replicas produce one ``manifest_sha256``. +VOLATILE_COMPUTE_FIELDS: tuple[str, ...] = ( + "wall_clock_seconds", + "peak_rss_bytes", + "peak_vram_bytes", +) + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def stage_tiny_train_data(root: Path | str, *, lines: int = 64) -> Path: + """Stage a tiny locked train shard under ``root`` and return its directory. + + Mirrors the locked FineWeb-Edu train mount the real broker provides, sized so a byte-level run + covers several instrument batches deterministically. + """ + + data_dir = Path(root) / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + shard = data_dir / "train-00000.jsonl" + if not shard.exists(): + shard.write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def install_cpu_reexec_seam(train_data_dir: Path | str) -> None: + """Replace ``DockerExecutor.run`` with the CPU re-exec seam for this process. + + Idempotent: a repeated call re-binds the same CPU runner. No broker/Docker is ever contacted + after this is installed. + """ + + _container.DockerExecutor.run = cpu_reexec_run( # type: ignore[assignment,method-assign] + train_data_dir=Path(train_data_dir) + ) + + +def cpu_test_context(config: WorkerPlaneConfig) -> PrismContext: + """Build the tiny deterministic :class:`PrismContext` for CPU re-exec test mode.""" + + return PrismContext( + vocab_size=config.cpu_reexec_vocab_size, + sequence_length=config.cpu_reexec_sequence_length, + seed=config.cpu_reexec_seed, + step_budget=config.cpu_reexec_step_budget, + ) + + +def configure_cpu_reexec_test_mode(settings: Any) -> Path: + """Stage tiny train data (unless supplied) + install the CPU seam. Returns the train data dir. + + Called by :func:`create_app` when ``worker_plane.cpu_reexec_test_mode`` is on so the prism + service's own worker (e.g. the audit-replay path) re-executes on CPU, and reusable by a local + worker/validator agent process to install the exact same seam. + """ + + config = settings.worker_plane + if config.cpu_reexec_train_data_dir: + data_dir = Path(config.cpu_reexec_train_data_dir) + data_dir.mkdir(parents=True, exist_ok=True) + else: + data_dir = stage_tiny_train_data( + settings.base_eval_artifact_root, lines=config.cpu_reexec_train_lines + ) + install_cpu_reexec_seam(data_dir) + return data_dir + + +def normalize_manifest_for_replication(manifest: dict[str, Any]) -> dict[str, Any]: + """Return a deep copy with volatile ``compute`` timing/memory fields fixed to a constant. + + Two honest CPU re-execs of the SAME submission differ ONLY in these host-timing fields; fixing + them makes the canonical manifest bytes (and thus :func:`compute_manifest_sha256`) identical, + which is what lets two independent worker replicas agree and be accepted (rather than disputed). + """ + + normalized = copy.deepcopy(manifest) + compute = normalized.get("compute") + if isinstance(compute, dict): + for field in VOLATILE_COMPUTE_FIELDS: + if field in compute: + compute[field] = 0 + return normalized + + +@dataclass(frozen=True) +class CpuReexecOutcome: + """A CPU re-exec result: the (optionally normalized) manifest, its hash, and artifact dir.""" + + manifest: dict[str, Any] + manifest_sha256: str + artifact_dir: Path + + +def evaluate_cpu_reexec( + settings: Any, + *, + submission_id: str, + arch_code: str = TINY_ARCHITECTURE, + train_code: str = TINY_TRAINING, + normalize: bool = True, +) -> CpuReexecOutcome: + """Run one deterministic CPU re-exec for ``submission_id`` and hash its manifest. + + The CPU seam must already be installed (:func:`configure_cpu_reexec_test_mode`). Returns the + normalized-for-replication manifest and its ``manifest_sha256`` so a worker can sign a proof or + an auditor can compute the authoritative hash. + """ + + ctx = cpu_test_context(settings.worker_plane) + evaluator = PrismContainerEvaluator(settings=settings, ctx=ctx) + files = ( + SourceFile("architecture.py", arch_code, _sha256(arch_code)), + SourceFile("training.py", train_code, _sha256(train_code)), + ) + result = evaluator.evaluate( + submission_id=submission_id, + code=arch_code, + code_hash=files[0].sha256, + arch_hash=files[0].sha256, + backend=settings.execution_backend, + files=files, + ) + manifest = result.run_manifest + if manifest is None: + raise RuntimeError(f"CPU re-exec for {submission_id!r} produced no manifest") + if normalize: + manifest = normalize_manifest_for_replication(manifest) + return CpuReexecOutcome( + manifest=manifest, + manifest_sha256=compute_manifest_sha256(manifest), + artifact_dir=Path(result.artifact_output_path or ""), + ) + + +def _sha256(text: str) -> str: + from hashlib import sha256 + + return sha256(text.encode()).hexdigest() + + +__all__ = [ + "TINY_ARCHITECTURE", + "TINY_TRAINING", + "VOLATILE_COMPUTE_FIELDS", + "CpuReexecOutcome", + "configure_cpu_reexec_test_mode", + "cpu_test_context", + "evaluate_cpu_reexec", + "install_cpu_reexec_seam", + "normalize_manifest_for_replication", + "stage_tiny_train_data", +] diff --git a/tests/test_cpu_test_mode.py b/tests/test_cpu_test_mode.py new file mode 100644 index 0000000..f09db28 --- /dev/null +++ b/tests/test_cpu_test_mode.py @@ -0,0 +1,149 @@ +"""Explicit CPU re-exec test-mode config (VAL-PRISM-013/015; mission harness seam). + +Covers the opt-in ``worker_plane.cpu_reexec_test_mode`` wiring that installs the repo's own CPU +re-exec seam as configuration (no monkeypatch): the helper stages tiny locked train data, installs +``DockerExecutor.run`` -> CPU runner, authors a real ``prism_run_manifest.v2`` on CPU, normalizes +the volatile timing fields so two honest replicas of the SAME submission agree on one +``manifest_sha256``, and ``create_app`` drives it end-to-end to a recorded prequential-bpb score. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from conftest import signed_headers, two_script_bundle +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings +from prism_challenge.evaluator import container as _container +from prism_challenge.evaluator.cpu_test_mode import ( + TINY_ARCHITECTURE, + TINY_TRAINING, + VOLATILE_COMPUTE_FIELDS, + configure_cpu_reexec_test_mode, + evaluate_cpu_reexec, + normalize_manifest_for_replication, + stage_tiny_train_data, +) + + +@pytest.fixture(autouse=True) +def _restore_docker_run(): + """The seam is a process-wide class-attr swap; snapshot + restore so it never leaks.""" + + original = _container.DockerExecutor.run + try: + yield + finally: + _container.DockerExecutor.run = original + + +def _settings(tmp_path: Path, *, data_dir: Path | None = None) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'cpu.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane={ + "cpu_reexec_test_mode": True, + "cpu_reexec_train_data_dir": str(data_dir) if data_dir else None, + }, + ) + + +def test_configure_installs_seam_and_stages_data(tmp_path): + settings = _settings(tmp_path) + data_dir = configure_cpu_reexec_test_mode(settings) + + assert data_dir.is_dir() + assert (data_dir / "train-00000.jsonl").is_file() + # The seam is installed as an explicit config, not a test monkeypatch. + assert _container.DockerExecutor.run.__name__ == "_run" + + +def test_evaluate_cpu_reexec_authors_real_manifest(tmp_path): + settings = _settings(tmp_path) + configure_cpu_reexec_test_mode(settings) + + outcome = evaluate_cpu_reexec(settings, submission_id="sub-1") + + assert outcome.manifest["schema_version"] == "prism_run_manifest.v2" + assert outcome.manifest["run"]["device"] == "cpu" + assert len(outcome.manifest_sha256) == 64 + assert outcome.artifact_dir.is_dir() + + +def test_normalize_drops_volatile_compute_fields(): + manifest = {"compute": {"wall_clock_seconds": 12.5, "peak_rss_bytes": 999, "world_size": 1}} + normalized = normalize_manifest_for_replication(manifest) + for field in VOLATILE_COMPUTE_FIELDS: + if field in normalized["compute"]: + assert normalized["compute"][field] == 0 + # Non-volatile fields are preserved and the input is not mutated. + assert normalized["compute"]["world_size"] == 1 + assert manifest["compute"]["wall_clock_seconds"] == 12.5 + + +def test_two_replicas_of_same_submission_agree_on_hash(tmp_path): + # Two independent worker hosts (distinct artifact roots) re-exec the SAME submission; the + # normalized manifest hash MUST match so the base worker plane accepts (not disputes) them. + data_dir = stage_tiny_train_data(tmp_path / "shared") + settings_a = _settings(tmp_path / "a", data_dir=data_dir) + settings_b = _settings(tmp_path / "b", data_dir=data_dir) + configure_cpu_reexec_test_mode(settings_a) + outcome_a = evaluate_cpu_reexec(settings_a, submission_id="agree-1") + configure_cpu_reexec_test_mode(settings_b) + outcome_b = evaluate_cpu_reexec(settings_b, submission_id="agree-1") + + assert outcome_a.manifest_sha256 == outcome_b.manifest_sha256 + + +def test_create_app_test_mode_scores_without_monkeypatch(tmp_path): + settings = _settings(tmp_path) + db_path = tmp_path / "cpu.sqlite3" + with TestClient(create_app(settings)) as client: + payload = { + "code": two_script_bundle(arch_code=TINY_ARCHITECTURE, train_code=TINY_TRAINING), + "filename": "project.zip", + } + import json + + body = json.dumps(payload, separators=(",", ":")).encode() + response = client.post( + "/v1/submissions", + content=body, + headers={ + **signed_headers("secret", body, nonce="cpu-cfg"), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 200, response.text + submission_id = str(response.json()["id"]) + process = client.post( + "/internal/v1/worker/process-next", headers={"Authorization": "Bearer secret"} + ) + assert process.status_code == 200, process.text + status = client.get(f"/v1/submissions/{submission_id}").json() + assert status["status"] == "completed" + + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + assert row is not None + assert row[0] > 0.0 From ae0e3b35c77086d9fc769b382abce4f33c178153 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:41:35 +0000 Subject: [PATCH 10/14] feat(mission): divergence drill reconstructs dispute via /v1/workers/units Extend drill 3 to rebuild the full dispute->audit->invalidation->fault story from operator APIs alone (VAL-CROSS-011): (a)/(b) from the new master GET /v1/workers/units, (c) from prism GET /v1/submissions/{id}, (d) fault on GET /v1/workers + base worker status. Adds a master_units() signed-read helper. --- scripts/mission/launch.py | 69 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/scripts/mission/launch.py b/scripts/mission/launch.py index bf2c5c1..7c176fb 100644 --- a/scripts/mission/launch.py +++ b/scripts/mission/launch.py @@ -252,6 +252,12 @@ def master_active_workers(self, owner: str) -> list[dict[str, Any]]: resp.raise_for_status() return resp.json().get("workers", []) + def master_units(self) -> list[dict[str, Any]]: + headers = _master_signed_headers(self.validator_kp, "GET", "/v1/workers/units") + resp = httpx.get(f"{MASTER_URL}/v1/workers/units", headers=headers, timeout=10) + resp.raise_for_status() + return resp.json().get("units", []) + def worker_status_cli(self) -> str: cfg = self.workdir / "worker-cli.yaml" cfg.write_text(_worker_cli_yaml(MASTER_URL, WORKER_URIS["alice"]), encoding="utf-8") @@ -516,7 +522,7 @@ def drill_self_eval(h: Harness) -> bool: return passed -def drill_divergence(h: Harness) -> tuple[bool, dict[str, Any]]: +def drill_divergence(h: Harness) -> tuple[bool, bool, dict[str, Any]]: print("\n=== DRILL 3: divergence -> dispute -> audit -> fault (VAL-CROSS-003) ===") erin_hk = ss58(MINERS["erin"]) workers = [ @@ -549,6 +555,14 @@ def _fault_visible() -> dict[str, Any] | None: print(f" (d) prism submission {sid}: status={final.get('status')} score={_score_of(final)}") ok_no_live_score = _score_of(final) is None or final.get("status") != "completed" ok_fault_on_liar = fault["owner"] == ss58(MINERS["bob"]) + + # VAL-CROSS-011: reconstruct the whole dispute story from operator APIs alone + # (no DB/file reads): the new signed GET /v1/workers/units + prism submission + # status + fleet/CLI fault. + cross011_ok = _reconstruct_dispute_via_api( + h, sid=sid, fault=fault, submission=final, no_live_score=ok_no_live_score + ) + evidence = {"sid": sid, "fault": fault, "submission": final} for w in workers: h.kill(w) @@ -559,7 +573,55 @@ def _fault_visible() -> dict[str, Any] | None: f" RESULT: {'PASS' if passed else 'FAIL'} " f"(fault_on_liar={ok_fault_on_liar}, no_live_score={ok_no_live_score})" ) - return passed, evidence + return passed, cross011_ok, evidence + + +def _reconstruct_dispute_via_api( + h: Harness, *, sid: str, fault: dict[str, Any], submission: dict[str, Any], no_live_score: bool +) -> bool: + """VAL-CROSS-011: rebuild dispute -> audit -> invalidation -> fault via APIs only.""" + + print("\n=== DRILL 11: dispute lifecycle discoverable via APIs alone (VAL-CROSS-011) ===") + units = h.master_units() + unit = next( + ( + u + for u in units + if u.get("work_unit_id") == sid + or str(u.get("work_unit_id", "")).startswith(sid) + ), + None, + ) + # (a) the unit's disputed state is visible on the master surface. + ok_disputed = unit is not None and unit.get("status") == "disputed" + audit = (unit or {}).get("audit") or {} + # (b) the audit unit + validator executor kind + terminal outcome are visible. + ok_audit = ( + bool(audit) + and audit.get("executor_kind") == "validator" + and audit.get("outcome") in ("pending", "passed", "mismatch-resolved") + ) + # (c) the affected submission is not a live-ranked completed score. + ok_invalidated = no_live_score + # (d) the lying worker's fault is visible on GET /v1/workers AND base worker status. + cli = h.worker_status_cli() + ok_fault_both = bool(fault) and fault["worker_id"] in cli + print( + f" (a) GET /v1/workers/units unit={unit.get('work_unit_id') if unit else None} " + f"status={unit.get('status') if unit else None}" + ) + print( + f" (b) audit unit={audit.get('work_unit_id')} executor_kind={audit.get('executor_kind')} " + f"outcome={audit.get('outcome')}" + ) + print( + f" (c) prism submission status={submission.get('status')} " + f"no_live_score={ok_invalidated}" + ) + print(f" (d) fault worker={fault['worker_id']} visible on API+CLI={ok_fault_both}") + passed = ok_disputed and ok_audit and ok_invalidated and ok_fault_both + print(f" RESULT: {'PASS' if passed else 'FAIL'}") + return passed def drill_fleet_agreement(h: Harness) -> bool: @@ -640,8 +702,9 @@ def main() -> int: if "1" in selected: results["VAL-CROSS-001 pipeline"] = drill_full_pipeline(h) if "3" in selected: - ok, _ = drill_divergence(h) + ok, cross011_ok, _ = drill_divergence(h) results["VAL-CROSS-003 divergence"] = ok + results["VAL-CROSS-011 dispute-discoverable"] = cross011_ok if "2" in selected: results["VAL-CROSS-002 self-eval"] = drill_self_eval(h) if "6" in selected: From 9bd49239128d572958f922298c4ebab8fd38c038 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:55:41 +0000 Subject: [PATCH 11/14] docs(readme): describe worker plane (verify-only validators, admission rule, worker_plane flag) --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 159eb1b..fd2f38d 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,48 @@ See [Security model](docs/security.md) for the full anti-cheat and sandbox polic --- +## Worker Plane (verify-only validators) + +PRISM optionally moves its heavy GPU re-execution off validators and onto **miner-funded worker +agents** (deployed on Lium/Targon or a miner's own hardware via the BASE `base worker` CLI). When +this plane is on, validators run a **verify-only** pipeline instead of executing every submission. +The whole feature is gated behind the prism `worker_plane` config block (env prefix +`PRISM_WORKER_PLANE__`); with `worker_plane.enabled` **false** (the default) prism behaves exactly +as before — no `ExecutionProof` emission, no admission gate, legacy audit-free finalization. + +- **Worker-executed evaluation**: the master assigns each submission's gpu work unit to + miner-funded workers (R=2 distinct owners, never the submitter's own worker). Each worker + re-executes the miner's training loop under forced random init and posts the challenge-authored + `prism_run_manifest.v2.json` plus an **ExecutionProof** (tier 0 deterministic manifest hash + + worker sr25519 signature; tier 1 adds a matching pinned image digest; tier 2 in-guest + attestation is schema-only and gated off on Targon). The master reconciles the replicas by + manifest hash and forwards one reconciled result; prism then **finalizes the prequential + bits-per-byte score from the forwarded manifest without re-executing** the evaluation. +- **Verify-only validator pipeline**: validators no longer run every submission. They run + **plausibility checks** (manifest schema/version, loss-trajectory sanity vs the random-init + baseline, step-0 anomaly, wall-clock vs declared budget) plus a **probabilistic, tier-modulated + audit scheduler** that samples finalized results and replays them deterministically through the + existing validator dispatch path (now audit-only). Tier-dependent sampling rates default to + `audit_rate_tier0=0.10`, `audit_rate_tier1=0.05`, `audit_rate_tier2=0.02`; a secret + `audit_salt` makes selection unpredictable from the public `submission_id`. An audit mismatch + invalidates the submission's score and advances architecture-family ownership to the surviving + best submission. +- **Admission rule**: with `worker_plane.admission_requires_worker` on, `POST /v1/submissions` + (and the BASE bridge path) is rejected with `403 NO_ACTIVE_WORKER` unless the master's + `GET /v1/workers/active?hotkey=` confirms ≥1 active worker bound to the submitting hotkey. A + master that is unreachable/slow folds into the same fail-closed rejection (bounded by + `admission_timeout_seconds`). Flag off ⇒ the check is a no-op. +- **Trust model unchanged**: the secret held-out `val`/`test` splits and the LLM provider keys + never reach workers; worker results are untrusted until reconciled (R=2) or audited. + +Config lives in `worker_plane` in [`config.example.yaml`](config.example.yaml) (keys `enabled`, +`admission_requires_worker`, `master_base_url`, `admission_timeout_seconds`, `audit_rate_tier{0,1,2}`, +`audit_salt`, `pinned_image_digest`). See the BASE +[Miner worker deployment guide](https://github.com/BaseIntelligence/base/blob/main/docs/miner/worker-plane.md) +for how miners deploy the workers that execute these units. + +--- + ## Repository Layout ```text From d037aba35e6f259af36af8b7f8ca64c7142726e8 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:38:41 +0000 Subject: [PATCH 12/14] test(mission): flags-OFF legacy validator-flow smoke (VAL-CROSS-006) Add a legacy validator that runs an assigned prism PRIMARY unit through the real validator_dispatch path, plus a flags-OFF smoke orchestrator that stands up the local mock-metagraph deployment with the worker plane OFF and drives one legacy submission end-to-end. It asserts (via HTTP + master/prism DB reads) the pre-mission behavior: submission accepted with no admission 403, exactly one gpu work unit, the unit assigned to the VALIDATOR (never a worker) with no worker-plane rows/side effects, and a score finalized via validator_dispatch. Wire a worker_plane_enabled toggle into the mission prism service so the flag OFF disables the worker plane + admission gate. --- scripts/mission/legacy_smoke.py | 287 ++++++++++++++++++++ scripts/mission/mission_legacy_validator.py | 155 +++++++++++ scripts/mission/mission_prism.py | 10 +- 3 files changed, 450 insertions(+), 2 deletions(-) create mode 100644 scripts/mission/legacy_smoke.py create mode 100644 scripts/mission/mission_legacy_validator.py diff --git a/scripts/mission/legacy_smoke.py b/scripts/mission/legacy_smoke.py new file mode 100644 index 0000000..fba5482 --- /dev/null +++ b/scripts/mission/legacy_smoke.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +"""Flags-OFF legacy regression smoke: gpu unit -> VALIDATOR -> validator_dispatch -> score. + +Part of the cross-repo local end-to-end harness (base ``docs/operations/mission-harness.md``); +fulfils part (b) of VAL-CROSS-006. Stands up the SAME local mock-metagraph deployment as +``launch.py`` but with ALL new flags OFF: + +* base master with ``worker_plane_enabled: false`` (gpu units route to gpu VALIDATORS, byte-for-byte + as pre-mission -- VAL-MASTER-013), and +* prism with ``worker_plane_enabled: false`` (admission gate off, ``/internal/v1/work_units/result`` + + audit routes inert), + +then drives one legacy submission end-to-end and asserts, via HTTP + a read of the master/prism +databases, that it behaves exactly as today: + +1. the submission is ACCEPTED with no ``NO_ACTIVE_WORKER`` 403 (admission off); +2. prism exposes exactly ONE gpu work unit for it; +3. the base master assigns that unit to the VALIDATOR + (``work_assignments.assigned_validator_hotkey`` == the validator hotkey, + ``required_capability == gpu``) and NEVER to a worker (``worker_assignments`` and + ``worker_registrations`` are empty -- no worker-plane rows/side effects); +4. the validator executes it via the real ``validator_dispatch`` path (its log shows the dispatch) + and prism records a score (``GET /v1/submissions/{id}`` -> completed with a final_score). + +Every spawned process' PID is printed and, in a ``finally``, KILLED by PID so nothing is left +listening. Run through the prism virtualenv with the current base source on ``PYTHONPATH``:: + + cd # contains base/ and prism/ + export PYTHONPATH="$PWD/base/src:$PWD/prism/src" + prism/.venv/bin/python prism/scripts/mission/legacy_smoke.py + +NOT for production. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any + +import httpx + +# Reuse the harness helpers (signed headers, process mgmt, waiters) from the main launcher. +from launch import ( # type: ignore[import-not-found] + BASE_MASTER_SCRIPT, + PRISM_SCRIPT_DIR, + Harness, + _completed_with_score, + _master_signed_headers, + _prism_signed_headers, + _score_of, + _two_script_bundle, + ss58, + wait_health, + wait_until, +) + +from prism_challenge.evaluator.cpu_test_mode import TINY_ARCHITECTURE, TINY_TRAINING + +MASTER_PORT = 3112 +PRISM_PORT = 3122 +MASTER_URL = f"http://127.0.0.1:{MASTER_PORT}" +PRISM_URL = f"http://127.0.0.1:{PRISM_PORT}" +TOKEN = "mission-shared-token" +NETUID = 100 +MINER_URI = "//MissionAlice" +VALIDATOR_URI = "//MissionValidator1" + + +class LegacyHarness(Harness): + """A flags-OFF variant of the harness: legacy master + prism + a legacy validator.""" + + def __init__(self, workdir: Path) -> None: + super().__init__(workdir=workdir) + self.prism_db = self.workdir / "prism.sqlite3" + self.master_db = self.workdir / "master.sqlite3" + + def start_master(self): # type: ignore[override] + entries = [ + {"hotkey": ss58(MINER_URI), "uid": 0, "validator_permit": False, "stake": 1000.0}, + {"hotkey": ss58(VALIDATOR_URI), "uid": 99, "validator_permit": True, "stake": 5000.0}, + ] + config = { + "port": MASTER_PORT, + "host": "127.0.0.1", + "db_url": f"sqlite+aiosqlite:///{self.master_db}", + "netuid": NETUID, + "metagraph": entries, + "prism": {"slug": "prism", "internal_base_url": PRISM_URL, "token": TOKEN}, + "orchestration_interval_seconds": 1.0, + "worker_heartbeat_ttl_seconds": 30, + "health_interval_seconds": 2.0, + "worker_plane_enabled": False, + } + return self.spawn("master", BASE_MASTER_SCRIPT, config) + + def start_prism(self): # type: ignore[override] + config = { + "port": PRISM_PORT, + "host": "127.0.0.1", + "db_path": str(self.prism_db), + "token": TOKEN, + "master_base_url": MASTER_URL, + "artifact_root": str(self.workdir / "prism-artifacts"), + "train_data_dir": str(self.train_dir), + "worker_plane_enabled": False, + "sequence_length": 16, + } + return self.spawn("prism", PRISM_SCRIPT_DIR / "mission_prism.py", config) + + def start_legacy_validator(self): + config = { + "master_url": MASTER_URL, + "validator_uri": VALIDATOR_URI, + "capabilities": ["gpu"], + "version": "0.1.0", + "heartbeat_interval_seconds": 3, + "poll_interval_seconds": 1.0, + "prism": { + "token": TOKEN, + "db_path": str(self.prism_db), + "artifact_root": str(self.workdir / "legacy-validator-artifacts"), + "train_data_dir": str(self.train_dir), + "sequence_length": 16, + }, + } + return self.spawn( + "legacy-validator", PRISM_SCRIPT_DIR / "mission_legacy_validator.py", config + ) + + # -- HTTP + DB observation helpers --------------------------------------- + def submit(self, *, nonce: str) -> httpx.Response: + code = _two_script_bundle(TINY_ARCHITECTURE, TINY_TRAINING) + import json + + body = json.dumps({"code": code, "filename": "project.zip"}, separators=(",", ":")).encode() + headers = { + **_prism_signed_headers(TOKEN, body, hotkey=ss58(MINER_URI), nonce=nonce), + "Content-Type": "application/json", + } + return httpx.post(f"{PRISM_URL}/v1/submissions", content=body, headers=headers, timeout=15) + + def prism_submission(self, sid: str) -> dict[str, Any]: + return httpx.get(f"{PRISM_URL}/v1/submissions/{sid}", timeout=10).json() + + def prism_work_units(self) -> list[dict[str, Any]]: + resp = httpx.get( + f"{PRISM_URL}/internal/v1/work_units", + headers={"Authorization": f"Bearer {TOKEN}", "X-Base-Challenge-Slug": "prism"}, + timeout=10, + ) + resp.raise_for_status() + return resp.json().get("work_units", []) + + def master_workers_status(self) -> tuple[int, Any]: + headers = _master_signed_headers(self.validator_kp, "GET", "/v1/workers") + resp = httpx.get(f"{MASTER_URL}/v1/workers", headers=headers, timeout=10) + try: + body: Any = resp.json() + except Exception: + body = resp.text + return resp.status_code, body + + def work_assignment_row(self, work_unit_id: str) -> dict[str, Any] | None: + con = sqlite3.connect(str(self.master_db)) + con.row_factory = sqlite3.Row + try: + row = con.execute( + "SELECT work_unit_id, submission_ref, required_capability, status, " + "assigned_validator_hotkey FROM work_assignments WHERE work_unit_id = ?", + (work_unit_id,), + ).fetchone() + finally: + con.close() + return dict(row) if row is not None else None + + def _count(self, table: str) -> int: + con = sqlite3.connect(str(self.master_db)) + try: + return int(con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + finally: + con.close() + + +def _enable_wal(db_path: Path) -> str: + con = sqlite3.connect(str(db_path)) + try: + mode = con.execute("PRAGMA journal_mode=WAL").fetchone()[0] + finally: + con.close() + return str(mode) + + +def run(workdir: Path) -> bool: + h = LegacyHarness(workdir=workdir) + results: dict[str, bool] = {} + try: + print("== bring up LEGACY (flags OFF) master + prism ==") + h.start_master() + h.start_prism() + wait_health(MASTER_URL, "master") + wait_health(PRISM_URL, "prism") + # WAL so the prism service (reader) and the legacy validator (single writer) never contend. + print(f" prism db journal_mode -> {_enable_wal(h.prism_db)}") + + print("\n== start the LEGACY validator (real validator_dispatch executor) ==") + h.start_legacy_validator() + time.sleep(5) # let it register + heartbeat before work exists + + print("\n=== (1) admission OFF: submission accepted with no NO_ACTIVE_WORKER 403 ===") + resp = h.submit(nonce="legacy-1") + print(f" POST /v1/submissions -> HTTP {resp.status_code} body={resp.text[:160]}") + ok_accept = resp.status_code < 300 and "NO_ACTIVE_WORKER" not in resp.text + results["(1) submission accepted, no admission 403"] = ok_accept + sid = str(resp.json()["id"]) if ok_accept else "" + print(f" submission id = {sid}") + + print("\n=== (2) prism exposes exactly one gpu work unit ===") + units = wait_until( + "prism gpu work unit", + lambda: [u for u in h.prism_work_units() if str(u.get("submission_id")) == sid], + timeout=30, + ) + cap = units[0].get("required_capability") or units[0].get("capability") + print(f" /internal/v1/work_units -> {len(units)} unit(s); required_capability={cap}") + results["(2) exactly one gpu work unit"] = len(units) == 1 + + print("\n=== (4) validator executes via validator_dispatch and prism records a score ===") + final = wait_until( + "prism records a score", + lambda: _completed_with_score(h.prism_submission(sid)), + timeout=180, + ) + print(f" GET /v1/submissions/{sid}: status={final.get('status')} score={_score_of(final)}") + results["(4) submission scored via validator_dispatch"] = _score_of(final) is not None + + print("\n=== (3) routing: gpu unit assigned to VALIDATOR, never a worker ===") + row = wait_until( + "work_assignments row for the unit", + lambda: h.work_assignment_row(sid), + timeout=30, + ) + validator_hk = ss58(VALIDATOR_URI) + print( + f" work_assignments: work_unit_id={row['work_unit_id']} " + f"required_capability={row['required_capability']} status={row['status']} " + f"assigned_validator_hotkey={row['assigned_validator_hotkey']}" + ) + assigned_to_validator = ( + row["assigned_validator_hotkey"] == validator_hk + and row["required_capability"] == "gpu" + ) + worker_regs = h._count("worker_registrations") + worker_asgn = h._count("worker_assignments") + print(f" worker_registrations rows={worker_regs} worker_assignments rows={worker_asgn}") + status_code, workers_body = h.master_workers_status() + n_workers = len(workers_body.get("workers", [])) if isinstance(workers_body, dict) else 0 + print(f" GET /v1/workers -> HTTP {status_code} workers={n_workers}") + results["(3) assigned to validator, not a worker"] = assigned_to_validator + results["(3) no worker-plane rows/side effects"] = worker_regs == 0 and worker_asgn == 0 + results["(3) worker fleet surface inert (no workers)"] = ( + status_code == 404 or n_workers == 0 + ) + finally: + h.kill_all() + + print("\n==================== LEGACY SMOKE SUMMARY (flags OFF) ====================") + for name, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL'} {name}") + all_ok = bool(results) and all(results.values()) + print("=========================================================================") + print(f" OVERALL: {'PASS' if all_ok else 'FAIL'}") + return all_ok + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workdir", default="/tmp/mission-legacy-smoke") + args = parser.parse_args() + return 0 if run(Path(args.workdir)) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mission/mission_legacy_validator.py b/scripts/mission/mission_legacy_validator.py new file mode 100644 index 0000000..5e586fd --- /dev/null +++ b/scripts/mission/mission_legacy_validator.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +"""Local mission LEGACY validator: run an assigned prism PRIMARY unit via ``validator_dispatch``. + +Part of the cross-repo local end-to-end harness (base ``docs/operations/mission-harness.md``); used +by the flags-OFF legacy regression smoke (VAL-CROSS-006). Enrolls as a gpu-capable validator (its +hotkey holds a validator permit in the mock metagraph). With the worker plane OFF the base master +routes the single prism gpu work unit to this VALIDATOR (never a worker); the agent pulls it and +runs the REAL prism ``validator_dispatch`` path (``dispatch_assignment`` -> +``run_primary_execution_cycle``) on the deterministic CPU re-exec seam, finalizing the score in the +SHARED prism database exactly as a pre-mission decentralized validator would. + +The base master (mock metagraph) mints no scoped gateway token in this harness, so a no-op gateway +token is injected here BEFORE dispatch (prism LLM review is disabled, so it is never used) purely to +satisfy the primary-path gateway config; nothing is sent to any real gateway. + +CONFIG-DRIVEN (JSON path in ``argv[1]`` / ``$MISSION_LEGACY_VALIDATOR_CONFIG``). NOT for production. +""" + +from __future__ import annotations + +import asyncio +import json +import signal +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import bittensor as bt +from base.validator.agent import BrokerConfig, CoordinationClient, ValidatorAgent +from base.validator.agent.adapters.prism import PrismCycleExecutor +from base.validator.agent.challenge_dispatch import ChallengeDispatchExecutor +from base.validator.agent.signing import KeypairRequestSigner + +from prism_challenge.config import PrismSettings +from prism_challenge.evaluator.cpu_test_mode import configure_cpu_reexec_test_mode +from prism_challenge.validator_dispatch import CHALLENGE_SLUG, dispatch_assignment + +# A no-op gateway token/URL: the primary dispatch path requires a scoped gateway config, but the +# prism LLM review is disabled in this harness so the token is never used and no gateway is called. +_NOOP_GATEWAY_TOKEN = "mission-legacy-noop-token" +_NOOP_GATEWAY_URL = "http://127.0.0.1:3199/llm/v1" + + +def _load_config() -> dict[str, Any]: + path = sys.argv[1] if len(sys.argv) > 1 else None + if path is None: + import os + + path = os.environ.get("MISSION_LEGACY_VALIDATOR_CONFIG") + if not path: + raise SystemExit("usage: mission_legacy_validator.py ") + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def _legacy_settings(prism: dict[str, Any]) -> PrismSettings: + """Prism settings for the legacy validator: SHARED db, worker plane OFF, CPU re-exec seam.""" + + token = prism["token"] + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{prism['db_path']}", + shared_token=token, + shared_token_file=None, + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + plagiarism_enabled=False, + distributed_contract_policy="off", + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://mission-broker:8082", + docker_broker_token=token, + sequence_length=int(prism.get("sequence_length", 16)), + base_eval_artifact_root=Path(prism["artifact_root"]), + worker_plane={ + "enabled": False, + "cpu_reexec_test_mode": True, + "cpu_reexec_train_data_dir": prism["train_data_dir"], + "cpu_reexec_sequence_length": int(prism.get("sequence_length", 16)), + }, + ) + + +def _build_dispatch(settings: PrismSettings): + """Bind the real prism dispatch to the legacy settings + a no-op gateway token.""" + + async def _dispatch( + *, + work_unit_id: str, + payload: Mapping[str, Any], + broker_url: str, + broker_token: str | None = None, + broker_token_file: str | None = None, + ) -> Mapping[str, Any]: + print( + f"[legacy-validator] pulled + executing prism unit {work_unit_id} " + "via validator_dispatch", + flush=True, + ) + enriched = dict(payload) + enriched.setdefault("gateway_token", _NOOP_GATEWAY_TOKEN) + enriched.setdefault("gateway_url", _NOOP_GATEWAY_URL) + result = await dispatch_assignment( + work_unit_id=work_unit_id, + payload=enriched, + broker_url=broker_url, + broker_token=broker_token, + broker_token_file=broker_token_file, + settings=settings, + ) + print(f"[legacy-validator] dispatch result for {work_unit_id}: {dict(result)}", flush=True) + return result + + return _dispatch + + +async def _run(config: dict[str, Any]) -> None: + prism = config["prism"] + Path(prism["artifact_root"]).mkdir(parents=True, exist_ok=True) + settings = _legacy_settings(prism) + configure_cpu_reexec_test_mode(settings) + + validator_kp = bt.Keypair.create_from_uri(config["validator_uri"]) + client = CoordinationClient(config["master_url"], KeypairRequestSigner(validator_kp)) + executor = ChallengeDispatchExecutor( + executors={CHALLENGE_SLUG: PrismCycleExecutor(dispatch=_build_dispatch(settings))} + ) + agent = ValidatorAgent( + client=client, + executor=executor, + broker=BrokerConfig(broker_url="http://mission-broker:8082"), + capabilities=config.get("capabilities", ["gpu"]), + version=config.get("version", "0.1.0"), + gateway_url=config["master_url"], + heartbeat_interval_seconds=int(config.get("heartbeat_interval_seconds", 3)), + poll_interval_seconds=float(config.get("poll_interval_seconds", 1.0)), + ) + + shutdown = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, shutdown.set) + except (NotImplementedError, RuntimeError): + pass + await agent.run_forever(shutdown) + + +def main() -> None: + asyncio.run(_run(_load_config())) + + +if __name__ == "__main__": + main() diff --git a/scripts/mission/mission_prism.py b/scripts/mission/mission_prism.py index 5207dda..d0b62ca 100644 --- a/scripts/mission/mission_prism.py +++ b/scripts/mission/mission_prism.py @@ -40,6 +40,11 @@ def _load_config() -> dict[str, Any]: def build_settings(config: dict[str, Any]) -> PrismSettings: token = config["token"] + # Legacy regression mode (VAL-CROSS-006): worker plane OFF + admission OFF so the service + # behaves byte-for-byte as pre-mission (submissions accepted with no active worker; the + # /internal/v1/work_units/result + audit routes are inert/404). The CPU re-exec seam stays + # available for whichever executor runs (a legacy validator dispatch, in this mode). + worker_plane_enabled = bool(config.get("worker_plane_enabled", True)) return PrismSettings( database_url=f"sqlite+aiosqlite:///{config['db_path']}", shared_token=token, @@ -58,8 +63,9 @@ def build_settings(config: dict[str, Any]) -> PrismSettings: base_eval_artifact_root=Path(config["artifact_root"]), public_submissions_enabled=True, worker_plane={ - "enabled": True, - "admission_requires_worker": bool(config.get("admission_requires_worker", True)), + "enabled": worker_plane_enabled, + "admission_requires_worker": worker_plane_enabled + and bool(config.get("admission_requires_worker", True)), "master_base_url": config["master_base_url"], "cpu_reexec_test_mode": True, "cpu_reexec_train_data_dir": config.get("train_data_dir"), From 2cfe4cf021d6718ffd8a162492b937e903881039 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:11:31 +0000 Subject: [PATCH 13/14] fix(prism): distinguishable retryable finalize failure + per-audit claim lease finalize_worker_result no longer returns a truthy submission_id when the source-static derivation FAILS: it reverts the CAS claim to pending (retryable) and raises WorkerFinalizationError, which ingestion surfaces as a distinct finalization_failed outcome (finalized=False, nothing recorded, HTTP 503) so a transient internal error is retried instead of being silently marked terminally finalized (status=failed). Happy path and all VAL-FINAL assertions preserved. run_validator_audit_cycle now claims each pending audit under a lightweight per-audit lease (repository.claim_audit_unit + audit_units.claimed_at/by, audit_claim_lease_seconds) before replaying it, so a multi-validator deployment no longer has every validator redundantly replay the same audits; each pending audit is replayed by at most one validator. The claim is cleared on resolution so a re-audited unit is immediately reclaimable, and a crashed claimant's lease expires. Single-validator behavior and VAL-FINAL-005 unchanged. --- src/prism_challenge/app.py | 9 +- src/prism_challenge/config.py | 6 + src/prism_challenge/db.py | 7 + src/prism_challenge/ingestion.py | 18 +- src/prism_challenge/queue.py | 42 +- src/prism_challenge/repository.py | 38 +- src/prism_challenge/validator_executor.py | 23 +- tests/test_prism_finalization_robustness.py | 456 ++++++++++++++++++++ tests/test_prism_result_ingestion.py | 2 +- 9 files changed, 589 insertions(+), 12 deletions(-) create mode 100644 tests/test_prism_finalization_robustness.py diff --git a/src/prism_challenge/app.py b/src/prism_challenge/app.py index 8bb6cdc..5aeb633 100644 --- a/src/prism_challenge/app.py +++ b/src/prism_challenge/app.py @@ -272,8 +272,15 @@ async def work_unit_result(request: Request) -> dict[str, object]: audit_sampler=sampler, ) except ResultIngestionError as exc: + # A transient finalization failure is retryable -> 503 so the forwarder retries; the + # permanent rejections (bad proof / tampered / implausible manifest) stay 422. + code = ( + status.HTTP_503_SERVICE_UNAVAILABLE + if exc.reason == "finalization_failed" + else status.HTTP_422_UNPROCESSABLE_ENTITY + ) raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, + code, {"code": exc.reason, "detail": str(exc)}, ) from exc except PlausibilityError as exc: diff --git a/src/prism_challenge/config.py b/src/prism_challenge/config.py index 902e579..c8a012c 100644 --- a/src/prism_challenge/config.py +++ b/src/prism_challenge/config.py @@ -32,6 +32,12 @@ class WorkerPlaneConfig(BaseModel): audit_rate_tier0: float = Field(default=0.10, ge=0.0, le=1.0) audit_rate_tier1: float = Field(default=0.05, ge=0.0, le=1.0) audit_rate_tier2: float = Field(default=0.02, ge=0.0, le=1.0) + # Per-audit claim lease (seconds). The validator audit cycle claims each pending audit under + # this lease before replaying it, so in a MULTI-validator deployment each pending audit is + # replayed by at most one validator (idempotent-but-wasteful redundant GPU/CPU replays are + # avoided). A crashed claimant's lease expires and the audit becomes reclaimable; the default is + # generous enough to exceed a normal replay wall-time. Single-validator behaviour is unchanged. + audit_claim_lease_seconds: float = Field(default=1800.0, gt=0.0) # Server-side SECRET salt mixed into the audit sampler seed so audit selection cannot be # predicted from the public ``submission_id`` alone (a different salt selects a different set), # while staying reproducible for a fixed salt and preserving the per-tier rates (architecture.md diff --git a/src/prism_challenge/db.py b/src/prism_challenge/db.py index 91a0b9b..7304a81 100644 --- a/src/prism_challenge/db.py +++ b/src/prism_challenge/db.py @@ -192,6 +192,7 @@ "executor_kind TEXT NOT NULL DEFAULT 'validator', status TEXT NOT NULL," "attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL DEFAULT 3," "resolved_manifest_sha256 TEXT, resolution TEXT, error TEXT," + "claimed_at TEXT, claimed_by TEXT," "created_at TEXT NOT NULL, updated_at TEXT NOT NULL);" "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" @@ -346,10 +347,16 @@ async def _run_migrations(conn: aiosqlite.Connection) -> None: "executor_kind TEXT NOT NULL DEFAULT 'validator', status TEXT NOT NULL," "attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL DEFAULT 3," "resolved_manifest_sha256 TEXT, resolution TEXT, error TEXT," + "claimed_at TEXT, claimed_by TEXT," "created_at TEXT NOT NULL, updated_at TEXT NOT NULL);" "CREATE INDEX IF NOT EXISTS idx_audit_units_submission ON audit_units(submission_id);" "CREATE INDEX IF NOT EXISTS idx_audit_units_status ON audit_units(status);" ) + await _ensure_columns( + conn, + "audit_units", + {"claimed_at": "TEXT", "claimed_by": "TEXT"}, + ) await conn.executescript( "CREATE TABLE IF NOT EXISTS worker_faults (" "id INTEGER PRIMARY KEY AUTOINCREMENT, audit_unit_id TEXT NOT NULL," diff --git a/src/prism_challenge/ingestion.py b/src/prism_challenge/ingestion.py index 95686cc..4d5d683 100644 --- a/src/prism_challenge/ingestion.py +++ b/src/prism_challenge/ingestion.py @@ -44,7 +44,7 @@ compute_manifest_sha256, verify_execution_proof, ) -from .queue import PrismWorker +from .queue import PrismWorker, WorkerFinalizationError logger = logging.getLogger(__name__) @@ -85,7 +85,10 @@ class ResultIngestionError(Exception): * ``signature_invalid`` -- the worker signature does not verify for this unit (VAL-PRISM-007b/c); * ``manifest_missing`` -- the worker plane is on but no run manifest was forwarded, so there is - nothing to finalize from without re-executing (VAL-FINAL-001). + nothing to finalize from without re-executing (VAL-FINAL-001); + * ``finalization_failed`` -- worker-plane finalization failed for an internal/transient reason + (source-static derivation error): the submission is reverted to pending and NOTHING is + recorded, so the forwarded result is retried rather than sealed as a clean finalize. """ def __init__(self, reason: str, message: str = "") -> None: @@ -290,7 +293,16 @@ async def ingest_work_unit_result( "manifest_missing", "worker-plane finalization requires the forwarded run manifest", ) - result_id = await worker.finalize_worker_result(submission_id, dict(manifest)) + try: + result_id = await worker.finalize_worker_result(submission_id, dict(manifest)) + except WorkerFinalizationError as exc: + # An internal/transient derivation failure is NOT a clean finalize: nothing is recorded + # (so a redelivery is genuinely retried, not idempotent-skipped) and the submission was + # reverted to pending. Surface it with a distinct, retryable reason. + raise ResultIngestionError( + "finalization_failed", + f"worker-plane finalization failed transiently and is retryable: {exc}", + ) from exc else: # Flag OFF: legacy in-process re-execution finalization, byte-for-byte unchanged. result_id = await worker.process_submission(submission_id) diff --git a/src/prism_challenge/queue.py b/src/prism_challenge/queue.py index c858600..a95002d 100644 --- a/src/prism_challenge/queue.py +++ b/src/prism_challenge/queue.py @@ -69,6 +69,17 @@ class EvalWallTimeExceeded(RuntimeError): """ +class WorkerFinalizationError(RuntimeError): + """Raised when worker-plane finalization cannot derive its score inputs from the submission. + + Deriving the deterministic source-static tail (snapshot -> component review -> anti-cheat) from + the submission SOURCE is an internal prism step, not a worker fault; a failure here is transient + and MUST NOT look like a clean finalize. The claimed submission is reverted to ``pending`` + (retryable) and this signals ingestion to report the forwarded result as un-finalized rather + than terminally failed, so a transient derivation error is retried instead of silently sealed. + """ + + EvaluatorFactory = Callable[[PrismSettings, PrismContext], PrismContainerEvaluator] @@ -178,6 +189,11 @@ async def finalize_worker_result( ``_evaluate_within_wall_time`` / ``_augment_with_heldout``). The held-out delta is SKIPPED (``skip_heldout=True``) so the score is bpb-only and the master-only secret val split (``base_eval_val_data_dir``) is never read (architecture.md 4). + + A failure while deriving the source-static tail is transient and internal (not a worker + fault), so it does NOT terminally fail the submission: it reverts the claim to ``pending`` + and raises :class:`WorkerFinalizationError` so ingestion reports the result as un-finalized + and retryable rather than a clean finalize with ``status=failed``. """ submission = await self.repository.claim_submission(submission_id) if submission is None: @@ -204,8 +220,17 @@ async def finalize_worker_result( allowed_import_roots=self._local_import_roots(snapshot), ) except Exception as exc: - await self._fail_submission(submission_id, str(exc)) - return submission_id + # Deriving the score inputs from the submission SOURCE failed. This is an internal, + # transient prism error (not a worker fault), so it MUST NOT masquerade as a clean + # finalize: revert the CAS claim to pending so the forwarded result stays retryable and + # signal ingestion instead of terminally sealing the submission as failed. + logger.warning( + "worker-plane finalization could not derive score inputs for %s: %s", + submission_id, + exc, + ) + await self._revert_submission_to_pending(submission_id, str(exc)) + raise WorkerFinalizationError(str(exc)) from exc await self._finalize_container_score( submission_id=submission_id, arch_hash=arch_hash, @@ -821,6 +846,19 @@ async def _fail_submission(self, submission_id: str, reason: str) -> None: (SubmissionStatus.FAILED.value, reason, now_iso(), submission_id), ) + async def _revert_submission_to_pending(self, submission_id: str, reason: str) -> None: + """Return a claimed (running) submission to ``pending`` so its work unit stays retryable. + + Used when a worker-plane finalization fails for an internal/transient reason: the CAS claim + set the submission ``running``, and reverting it to ``pending`` (rather than terminally + ``failed``) lets a redelivered forwarded result re-claim and finalize it. + """ + async with self.repository.database.connect() as conn: + await conn.execute( + "UPDATE submissions SET status=?, error=?, updated_at=? WHERE id=?", + (SubmissionStatus.PENDING.value, reason, now_iso(), submission_id), + ) + def _review_rules(self) -> tuple[ReviewRule, ...]: return load_review_rules( defaults=DEFAULT_REVIEW_RULES, diff --git a/src/prism_challenge/repository.py b/src/prism_challenge/repository.py index cec0b1f..a955819 100644 --- a/src/prism_challenge/repository.py +++ b/src/prism_challenge/repository.py @@ -1179,6 +1179,34 @@ async def list_pending_audit_units(self) -> list[dict[str, object]]: ) return [dict(cast(Any, row)) for row in rows] + async def claim_audit_unit( + self, audit_unit_id: str, *, claimant: str, lease_seconds: float + ) -> bool: + """Atomically claim a pending audit unit under a lease; ``True`` when this caller won it. + + A single-consumer guard for the validator audit cycle: in a MULTI-validator deployment each + validator enumerates the same pending audits, so without a claim they would all redundantly + replay every one. The claim is an atomic CAS -- only the caller whose ``UPDATE`` matches + (the unit is still ``pending`` AND unclaimed or its lease has expired) stamps ``claimed_at`` + and wins; concurrent claimers get ``False`` and skip. The claim is orthogonal to the + lifecycle ``status`` (the unit stays ``pending``), so :func:`resolve_audit_unit` is + unaffected. A crashed claimant's lease expires (``claimed_at <= now - lease_seconds``) and + the audit becomes reclaimable, so a claim is never permanently orphaned. + """ + from .audit import AUDIT_STATUS_PENDING + + now = datetime.now(UTC) + now_str = now.isoformat() + cutoff = (now - timedelta(seconds=max(lease_seconds, 0.0))).isoformat() + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "UPDATE audit_units SET claimed_at=?, claimed_by=? " + "WHERE audit_unit_id=? AND status=? " + "AND (claimed_at IS NULL OR claimed_at <= ?) RETURNING audit_unit_id", + (now_str, claimant, audit_unit_id, AUDIT_STATUS_PENDING, cutoff), + ) + return bool(list(rows)) + async def record_audit_resolution( self, *, @@ -1189,11 +1217,17 @@ async def record_audit_resolution( resolved_manifest_sha256: str | None, error: str | None, ) -> None: - """Persist an audit unit's new lifecycle state after a resolution attempt.""" + """Persist an audit unit's new lifecycle state after a resolution attempt. + + The per-audit claim is cleared (``claimed_at``/``claimed_by`` -> NULL) on every resolution + so a unit returned to ``pending`` for a bounded re-audit is immediately reclaimable by any + validator rather than blocked until the previous claimant's lease expires. + """ async with self.database.connect() as conn: await conn.execute( "UPDATE audit_units SET status=?, attempts=?, resolution=?, " - "resolved_manifest_sha256=?, error=?, updated_at=? WHERE audit_unit_id=?", + "resolved_manifest_sha256=?, error=?, claimed_at=NULL, claimed_by=NULL, " + "updated_at=? WHERE audit_unit_id=?", ( status, int(attempts), diff --git a/src/prism_challenge/validator_executor.py b/src/prism_challenge/validator_executor.py index 2fef820..ac0a010 100644 --- a/src/prism_challenge/validator_executor.py +++ b/src/prism_challenge/validator_executor.py @@ -21,6 +21,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any +from uuid import uuid4 from .audit import AuditResolution, is_audit_unit_id, resolve_audit_unit from .coordination import ( @@ -300,33 +301,49 @@ async def run_validator_audit_cycle( worker: PrismWorker, work_unit_ids: Iterable[str] | None = None, audit_replay: AuditReplayFn | None = None, + claimant: str | None = None, ) -> PrismValidatorCycleSummary: """Execute the sampled prism AUDIT units assigned to this validator (architecture.md 3.5). Enumerates the pending ``audit:`` units (``list_pending_audit_units``, each id recognised via :func:`~prism_challenge.audit.is_audit_unit_id`), restricted to the caller's assigned - ``work_unit_ids`` when given (``None`` = every pending audit). For each, the audited + ``work_unit_ids`` when given (``None`` = every pending audit). Each candidate is CLAIMED under a + lightweight per-audit lease (``repository.claim_audit_unit``) BEFORE it is replayed, so in a + MULTI-validator deployment each pending audit is replayed by at most one validator instead of + every validator redundantly replaying the same set (idempotent but wasteful GPU/CPU). An audit + already claimed by another validator (live lease) is skipped, not replayed. Single-validator + behaviour is unchanged: the sole validator wins every claim. For each claimed audit, the audited submission's evaluation is replayed deterministically to obtain a fresh manifest sha256, and the replay result is resolved through :func:`~prism_challenge.audit.resolve_audit_unit` (the ``POST /internal/v1/audit_units/{id}/result`` target): a MATCHING hash passes (finalized score untouched); a DIVERGENT hash invalidates the score and records a ``worker_fault``; a replay failure resolves inconclusive (re-audited within bounds). This cycle NEVER executes a primary submission (VAL-FINAL-005). ``audit_replay`` defaults to the real container replay; tests inject - a deterministic hash. + a deterministic hash. ``claimant`` identifies the lease holder (defaults to a per-cycle id). """ replay = audit_replay if audit_replay is not None else worker.replay_audit_manifest_sha256 wanted = set(work_unit_ids) if work_unit_ids is not None else None + lease_seconds = worker.settings.worker_plane.audit_claim_lease_seconds + who = claimant or uuid4().hex pending = await worker.repository.list_pending_audit_units() resolutions: list[AuditResolution] = [] pulled = 0 executed = 0 + skipped = 0 for row in pending: audit_unit_id = str(row["audit_unit_id"]) if not is_audit_unit_id(audit_unit_id): continue if wanted is not None and audit_unit_id not in wanted: continue + if not await worker.repository.claim_audit_unit( + audit_unit_id, claimant=who, lease_seconds=lease_seconds + ): + # Another validator already holds a live claim on this audit: skip it rather than + # redundantly replay the same deterministic run (each audit is single-consumer). + skipped += 1 + continue pulled += 1 submission_id = str(row["submission_id"]) replay_hash: str | None @@ -349,7 +366,7 @@ async def run_validator_audit_cycle( return PrismValidatorCycleSummary( pulled=pulled, executed=executed, - skipped=0, + skipped=skipped, completed_submissions=(), audits=tuple(resolutions), ) diff --git a/tests/test_prism_finalization_robustness.py b/tests/test_prism_finalization_robustness.py new file mode 100644 index 0000000..42dbe93 --- /dev/null +++ b/tests/test_prism_finalization_robustness.py @@ -0,0 +1,456 @@ +"""Prism finalization robustness follow-ups (scrutiny prism-finalization). + +Two small robustness/efficiency guards on the sealed prism-finalization milestone: + +* Fix 1 -- ``PrismWorker.finalize_worker_result`` no longer masquerades an internal + SOURCE-derivation failure as a clean finalize. A derivation error (snapshot / component review / + anti-cheat) reverts the claimed submission to ``pending`` (retryable) and raises + ``WorkerFinalizationError``; ingestion surfaces it as a distinct ``finalization_failed`` outcome + (``finalized=False``, nothing recorded, the forwarded result can be retried) instead of + ``finalized=True`` with ``status=failed``. The happy path and every VAL-FINAL assertion hold. +* Fix 2 -- ``run_validator_audit_cycle`` claims each pending audit under a lightweight per-audit + lease so, in a MULTI-validator deployment, each pending audit is replayed by at most one validator + (idempotent but wasteful redundant GPU/CPU replays are avoided). Single-validator behaviour + (harness + live e2e) and VAL-FINAL-005 are unchanged. + +Offline, no GPU: worker-plane finalization scores from the forwarded manifest and audits use an +injected deterministic replay; proofs are signed with real sr25519 worker keys. +""" + +from __future__ import annotations + +import base64 +import io +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.audit import ( + AUDIT_STATUS_PASSED, + AUDIT_STATUS_PENDING, + AuditSampler, + audit_unit_id_for, +) +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.ingestion import ResultIngestionError, ingest_work_unit_result +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) +from prism_challenge.queue import PrismWorker, WorkerFinalizationError +from prism_challenge.validator_executor import run_validator_audit_cycle + +WORKER_KEY = "//WorkerRobustness" + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path, *, worker_plane: WorkerPlaneConfig) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'robust.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + llm_review_enabled=False, + llm_review_required=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=worker_plane, + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _proof_dict(signer, unit_id: str, manifest: dict[str, Any], **overrides: Any) -> dict[str, Any]: + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof(signer=signer, manifest_sha256=digest, unit_id=unit_id) + payload = proof.model_dump(mode="json") + payload.update(overrides) + return payload + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any] | None) -> dict[str, Any]: + result: dict[str, Any] = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + } + if manifest is not None: + result[MANIFEST_PAYLOAD_KEY] = manifest + return result + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _always() -> AuditSampler: + return AuditSampler(audit_rate_tier0=1.0, audit_rate_tier1=1.0, audit_rate_tier2=1.0) + + +async def _finalize_and_sample(app, signer, *, hotkey: str = "hk-owner") -> tuple[str, dict, Any]: + submission_id = await _seed(app, hotkey) + manifest = _manifest() + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=hotkey, + result=_result(_proof_dict(signer, submission_id, manifest), manifest), + audit_sampler=_always(), + ) + return submission_id, manifest, outcome + + +def _score(db_path: Path, submission_id: str): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return row[0] if row else None + + +# --- Fix 1: derivation failure is retryable, not a silent terminal finalize ---------------------- + + +async def test_finalize_worker_result_raises_on_derivation_failure(tmp_path, monkeypatch) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + repository = app.state.repository + + def _boom(self, snapshot): # noqa: ANN001, ANN202 + raise RuntimeError("transient derivation glitch") + + monkeypatch.setattr(PrismWorker, "_component_review", _boom) + + submission_id = await _seed(app) + with pytest.raises(WorkerFinalizationError): + await app.state.worker.finalize_worker_result(submission_id, _manifest()) + + # The claimed submission is reverted to pending (retryable), NOT terminally failed. + assert await repository.submission_status(submission_id) == "pending" + + +async def test_ingestion_reports_derivation_failure_as_retryable(tmp_path, monkeypatch) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + db_path = tmp_path / "robust.sqlite3" + + real_review = PrismWorker._component_review + calls = {"n": 0} + + def _flaky(self, snapshot): # noqa: ANN001, ANN202 + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("transient derivation glitch") + return real_review(self, snapshot) + + monkeypatch.setattr(PrismWorker, "_component_review", _flaky) + + submission_id = await _seed(app) + manifest = _manifest() + result = _result(_proof_dict(signer, submission_id, manifest), manifest) + + # First delivery: a derivation failure is a DISTINCT, retryable ingestion error -- NOT a clean + # finalize. Nothing is scored, the submission stays pending, and no work-unit result is recorded + # (so a redelivery is genuinely retried rather than idempotent-skipped). + with pytest.raises(ResultIngestionError) as exc: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=result, + ) + assert exc.value.reason == "finalization_failed" + assert await repository.submission_status(submission_id) == "pending" + assert _score(db_path, submission_id) is None + assert await repository.get_work_unit_result(submission_id) is None + + # Redelivery once the transient condition clears: the SAME forwarded result finalizes cleanly. + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=result, + ) + assert outcome.status == "accepted" + assert outcome.finalized is True + assert await repository.submission_status(submission_id) == "completed" + assert _score(db_path, submission_id) is not None + + +def test_result_route_returns_503_on_finalization_failure(tmp_path, monkeypatch) -> None: + from fastapi.testclient import TestClient + + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + signer = worker_signer_from_key(WORKER_KEY) + headers = {"Authorization": "Bearer secret"} + + def _boom(self, snapshot): # noqa: ANN001, ANN202 + raise RuntimeError("transient derivation glitch") + + monkeypatch.setattr(PrismWorker, "_component_review", _boom) + + with TestClient(create_app(settings)) as client: + seed = client.post( + "/internal/v1/bridge/submissions", + content=base64.b64decode(_bundle()), + headers={ + "Authorization": "Bearer secret", + "X-Base-Verified-Hotkey": "hk-owner", + "X-Submission-Filename": "project.zip", + "Content-Type": "application/octet-stream", + }, + ) + assert seed.status_code == 200, seed.text + submission_id = seed.json()["id"] + + manifest = _manifest() + body = { + "work_unit_id": submission_id, + "submission_ref": "hk-owner", + "result": _result(_proof_dict(signer, submission_id, manifest), manifest), + } + resp = client.post("/internal/v1/work_units/result", json=body, headers=headers) + # A transient internal derivation failure is retryable -> 503, distinct from the permanent + # 422 rejections (bad proof / implausible manifest) and the 409 conflict. + assert resp.status_code == 503, resp.text + assert resp.json()["detail"]["code"] == "finalization_failed" + + +# --- Fix 2: per-audit claim/lease so at most one validator replays each pending audit ------------- + + +async def test_claim_audit_unit_is_single_consumer_with_lease(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + submission_id, _, outcome = await _finalize_and_sample(app, signer) + audit_unit_id = outcome.audit_unit_id + assert audit_unit_id == audit_unit_id_for(submission_id) + + # The first validator wins the claim; a second validator with a live lease is refused. + assert await repository.claim_audit_unit( + audit_unit_id, claimant="validator-1", lease_seconds=10_000 + ) + assert not await repository.claim_audit_unit( + audit_unit_id, claimant="validator-2", lease_seconds=10_000 + ) + # The unit is still pending (claim is orthogonal to lifecycle status). + unit = await repository.get_audit_unit(audit_unit_id) + assert unit is not None + assert unit["status"] == AUDIT_STATUS_PENDING + assert unit["claimed_by"] == "validator-1" + + # An expired lease (cutoff == now) is reclaimable by another validator. + assert await repository.claim_audit_unit( + audit_unit_id, claimant="validator-3", lease_seconds=0 + ) + reclaimed = await repository.get_audit_unit(audit_unit_id) + assert reclaimed is not None + assert reclaimed["claimed_by"] == "validator-3" + + +async def test_audit_cycle_skips_units_claimed_by_another_validator(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + submission_id, manifest, outcome = await _finalize_and_sample(app, signer) + audit_unit_id = outcome.audit_unit_id + + # Another validator holds a live claim on the only pending audit. + assert await repository.claim_audit_unit( + audit_unit_id, claimant="other-validator", lease_seconds=10_000 + ) + + replays: list[str] = [] + + async def _replay(sub_id: str) -> str: + replays.append(sub_id) + return compute_manifest_sha256(manifest) + + summary = await run_validator_audit_cycle(worker=app.state.worker, audit_replay=_replay) + + # This validator never replays a claimed audit: no wasted GPU/CPU, nothing resolved. + assert replays == [] + assert summary.pulled == 0 + assert summary.executed == 0 + assert summary.skipped == 1 + assert summary.audits == () + still_pending = await repository.get_audit_unit(audit_unit_id) + assert still_pending is not None + assert still_pending["status"] == AUDIT_STATUS_PENDING + + +async def test_single_validator_audit_claims_replays_and_clears_claim(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "robust.sqlite3" + + submission_id, manifest, outcome = await _finalize_and_sample(app, signer) + score_before = _score(db_path, submission_id) + assert score_before is not None + + replays: list[str] = [] + + async def _replay(sub_id: str) -> str: + replays.append(sub_id) + return compute_manifest_sha256(manifest) + + # Single-validator behaviour is unchanged: the sole validator claims, replays, and passes. + summary = await run_validator_audit_cycle(worker=app.state.worker, audit_replay=_replay) + assert replays == [submission_id] + assert summary.pulled == 1 + assert summary.executed == 1 + assert summary.skipped == 0 + assert len(summary.audits) == 1 + assert summary.audits[0].status == AUDIT_STATUS_PASSED + assert _score(db_path, submission_id) == pytest.approx(score_before) + + # A terminal (passed) audit is no longer pending, so a second cycle bears no work. + again = await run_validator_audit_cycle(worker=app.state.worker, audit_replay=_replay) + assert again.pulled == 0 + assert again.executed == 0 + + +async def test_inconclusive_audit_claim_cleared_and_reclaimable(tmp_path) -> None: + settings = _settings( + tmp_path, worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY) + ) + app = await _make_app(settings) + signer = worker_signer_from_key(WORKER_KEY) + repository = app.state.repository + + submission_id, _, outcome = await _finalize_and_sample(app, signer) + audit_unit_id = outcome.audit_unit_id + + replays: list[str] = [] + + async def _inconclusive(sub_id: str) -> str | None: + replays.append(sub_id) + return None + + # An inconclusive replay (attempts < max) returns the audit to pending; the claim MUST be + # cleared so the re-audit is immediately reclaimable (by any validator, with a LIVE lease). + first = await run_validator_audit_cycle(worker=app.state.worker, audit_replay=_inconclusive) + assert first.pulled == 1 + reverted = await repository.get_audit_unit(audit_unit_id) + assert reverted is not None + assert reverted["status"] == AUDIT_STATUS_PENDING + assert reverted["claimed_at"] is None + + second = await run_validator_audit_cycle(worker=app.state.worker, audit_replay=_inconclusive) + assert second.pulled == 1 + assert replays == [submission_id, submission_id] diff --git a/tests/test_prism_result_ingestion.py b/tests/test_prism_result_ingestion.py index f543180..c18856a 100644 --- a/tests/test_prism_result_ingestion.py +++ b/tests/test_prism_result_ingestion.py @@ -500,7 +500,7 @@ def test_result_route_body_contract(tmp_path, monkeypatch) -> None: with TestClient(create_app(settings)) as client: seed = client.post( "/internal/v1/bridge/submissions", - content=_bundle().encode(), + content=base64.b64decode(_bundle()), headers={ "Authorization": "Bearer secret", "X-Base-Verified-Hotkey": "hk-owner", From cb8e9f29fc41b60259bac3cab0c36c0d30cfc533 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:01:42 +0000 Subject: [PATCH 14/14] test(mission): derive CROSS-001/002/009 drills from operator surfaces; harness log/re-run hygiene Strengthen the launch.py drills so PASS/FAIL derives strictly from operator- observable API/CLI surfaces: CROSS-001 asserts the finalized score via prism GET /v1/submissions/{id}; CROSS-002 asserts via GET /v1/workers/units that the submitter's own worker never became a replica of its unit; CROSS-009 diffs GET /v1/workers against parsed 'base worker status' (ids/owners/providers/ statuses/fault counts). Line-buffer + basicConfig mission_prism/mission_worker so drill logs survive SIGTERM teardown; use a random submission nonce per run so legacy_smoke (and the drills) are re-runnable against a reused --workdir. --- scripts/mission/launch.py | 184 ++++++++++++++++++++++-------- scripts/mission/legacy_smoke.py | 5 +- scripts/mission/mission_prism.py | 39 ++++++- scripts/mission/mission_worker.py | 37 ++++++ 4 files changed, 217 insertions(+), 48 deletions(-) diff --git a/scripts/mission/launch.py b/scripts/mission/launch.py index 7c176fb..029778b 100644 --- a/scripts/mission/launch.py +++ b/scripts/mission/launch.py @@ -27,6 +27,7 @@ import subprocess import sys import time +import uuid from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -424,14 +425,14 @@ def _ok() -> bool: def drill_admission(h: Harness) -> bool: print("\n=== DRILL 4: admission gate 403 -> acceptance after enrollment (VAL-CROSS-004) ===") - before = h.prism_submit("dave", nonce="adm-before") + before = h.prism_submit("dave", nonce=f"adm-before-{uuid.uuid4().hex}") print(f" before-enrollment submit: HTTP {before.status_code} body={before.text[:160]}") ok_403 = before.status_code == 403 and "NO_ACTIVE_WORKER" in before.text worker = h.start_worker("dave") wait_until("dave active worker", lambda: len(h.master_active_workers("dave")) >= 1, timeout=60) active = h.master_active_workers("dave") print(f" GET /v1/workers/active?hotkey=dave -> {len(active)} active worker(s)") - after = h.prism_submit("dave", nonce="adm-after") + after = h.prism_submit("dave", nonce=f"adm-after-{uuid.uuid4().hex}") after_id = after.json().get("id") if after.status_code < 300 else "-" print(f" after-enrollment submit: HTTP {after.status_code} id={after_id}") ok_after = after.status_code < 300 @@ -449,7 +450,7 @@ def drill_full_pipeline(h: Harness) -> bool: wait_until( f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 ) - resp = h.prism_submit("carol", nonce="pipe-1") + resp = h.prism_submit("carol", nonce=f"pipe-{uuid.uuid4().hex}") assert resp.status_code < 300, resp.text sid = str(resp.json()["id"]) print(f" (a) submission accepted id={sid}") @@ -462,7 +463,6 @@ def drill_full_pipeline(h: Harness) -> bool: f" (a) prism /internal/v1/work_units exposes {len(units)} unit(s) for {sid}, " f"submission_ref={units[0].get('submission_ref')}" ) - ok_unit = len(units) == 1 and units[0].get("submission_ref") == ss58(MINERS["carol"]) def _assigned_owners() -> set[str] | None: owners = { @@ -475,50 +475,80 @@ def _assigned_owners() -> set[str] | None: wait_until("2 active workers", _assigned_owners, timeout=30) print(" (b) fleet shows >=2 active distinct-owner workers") - final = wait_until( + wait_until( "prism records score", lambda: _completed_with_score(h.prism_submission(sid)), timeout=120 ) - print(f" (e) prism submission {sid}: status={final.get('status')} score={_score_of(final)}") - ok_score = _score_of(final) is not None + # Authoritative operator surface (VAL-CROSS-001): the finalized score is + # queryable via the prism submission read. PASS derives strictly from it. + final = h.prism_submission(sid) + score = _score_of(final) + print(f" (e) GET /v1/submissions/{sid}: status={final.get('status')} score={score}") + ok_score = final.get("status") == "completed" and score is not None for w in workers: h.kill(w) for owner in ("alice", "bob", "carol"): _wait_stale(h, ss58(MINERS[owner])) - passed = ok_unit and ok_score + passed = ok_score print(f" RESULT: {'PASS' if passed else 'FAIL'}") return passed def drill_self_eval(h: Harness) -> bool: print("\n=== DRILL 2: self-eval exclusion under scarcity (VAL-CROSS-002) ===") - # Exactly two active workers: alice (== submitter H) and bob. + # Exactly two active workers: alice (== submitter H) and bob (distinct owner). workers = [h.start_worker("alice"), h.start_worker("bob")] for owner in ("alice", "bob"): wait_until( f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 ) - resp = h.prism_submit("alice", nonce="self-1") + alice_hk = ss58(MINERS["alice"]) + alice_worker_ids = {w["worker_id"] for w in h.master_active_workers("alice")} + resp = h.prism_submit("alice", nonce=f"self-{uuid.uuid4().hex}") assert resp.status_code < 300, resp.text sid = str(resp.json()["id"]) - print(f" submission from H=alice accepted id={sid}") - alice_worker_id = h.master_active_workers("alice")[0]["worker_id"] - # Observe for a while that H's worker never becomes the ONE that finalizes; the unit is - # only ever handled by bob (or held). We assert the submission is not evaluated by alice's - # worker by confirming alice's worker records no fault and the score (if any) came from bob. - time.sleep(12) - final = h.prism_submission(sid) - print(f" after observation: submission status={final.get('status')} score={_score_of(final)}") - # Under R=1 scarcity prism may finalize from bob alone, or hold pending; either is acceptable - # provided alice's own worker never got the unit. We verify alice's worker id is stable/active - # and (weak API-only proxy) the pipeline never faulted alice. - faults = _faults_for_worker(h.master_workers(), alice_worker_id) - print(f" H(alice) worker_id={alice_worker_id} faults={len(faults)} (expected 0)") - passed = len(faults) == 0 + print(f" submission from H=alice accepted id={sid} own_worker_ids={sorted(alice_worker_ids)}") + + # Operator-observable assertion (VAL-CROSS-002): via GET /v1/workers/units, + # the submitter's OWN worker (owned by alice) must never appear as a replica + # of alice's unit while it waits/is handled. Under scarcity the only eligible + # distinct owner is bob, so the unit degrades to bob or holds pending -- never + # to alice. + replica_worker_ids: set[str] = set() + replica_owners: set[str] = set() + + def _alice_unit() -> dict[str, Any] | None: + found: dict[str, Any] | None = None + for unit in h.master_units(): + if unit.get("submission_ref") == alice_hk and unit.get("audit") is None: + found = unit + for replica in unit.get("replicas") or []: + if replica.get("worker_id"): + replica_worker_ids.add(replica["worker_id"]) + if replica.get("miner_hotkey"): + replica_owners.add(replica["miner_hotkey"]) + return found + + # The unit must really exist on the master surface (non-vacuous)... + wait_until("alice's unit visible via GET /v1/workers/units", _alice_unit, timeout=30) + # ...and across the observation window alice's own worker never becomes a replica. + deadline = time.time() + 15 + while time.time() < deadline: + _alice_unit() + time.sleep(1) + + self_became_replica = bool(replica_worker_ids & alice_worker_ids) or alice_hk in replica_owners + print( + f" GET /v1/workers/units alice-unit: replica_workers={sorted(replica_worker_ids)} " + f"replica_owners={sorted(replica_owners)} self_became_replica={self_became_replica}" + ) + passed = not self_became_replica for w in workers: h.kill(w) for owner in ("alice", "bob"): _wait_stale(h, ss58(MINERS[owner])) - print(f" RESULT: {'PASS (self-eval exclusion held)' if passed else 'FAIL'}") + print( + f" RESULT: {'PASS (self-eval exclusion held via /v1/workers/units)' if passed else 'FAIL'}" + ) return passed @@ -534,7 +564,7 @@ def drill_divergence(h: Harness) -> tuple[bool, bool, dict[str, Any]]: wait_until( f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 ) - resp = h.prism_submit("erin", nonce="div-1") + resp = h.prism_submit("erin", nonce=f"div-{uuid.uuid4().hex}") assert resp.status_code < 300, resp.text sid = str(resp.json()["id"]) print(f" submission from erin accepted id={sid} (bob will corrupt its manifest)") @@ -625,31 +655,100 @@ def _reconstruct_dispute_via_api( def drill_fleet_agreement(h: Harness) -> bool: - print("\n=== DRILL 6/9: fleet API vs CLI agree (VAL-CROSS-009) ===") + print("\n=== DRILL 6/9: fleet API vs `base worker status` CLI agree (VAL-CROSS-009) ===") workers = [h.start_worker("alice"), h.start_worker("bob")] for owner in ("alice", "bob"): wait_until( f"{owner} active", lambda o=owner: len(h.master_active_workers(o)) >= 1, timeout=60 ) - api_workers = h.master_workers() - api_ids = {w["worker_id"] for w in api_workers} - cli = h.worker_status_cli() - print(" --- GET /v1/workers (ids) ---") - for w in api_workers: + + # Diff the two operator surfaces: every worker's id, owner, provider, status, + # and fault count must be IDENTICAL between GET /v1/workers and the parsed + # `base worker status` CLI output (retried briefly so both surfaces observe + # the same heartbeat-derived status). + api_view: dict[str, dict[str, Any]] = {} + cli_view: dict[str, dict[str, Any]] = {} + cli_text = "" + agree = False + deadline = time.time() + 30 + while time.time() < deadline: + api_view = _fleet_from_api(h.master_workers()) + cli_text = h.worker_status_cli() + cli_view = _parse_cli_workers(cli_text) + if api_view and api_view == cli_view: + agree = True + break + time.sleep(2) + + print(" --- GET /v1/workers ---") + for wid, rec in sorted(api_view.items()): print( - f" {w['worker_id']} owner={w['miner_hotkey']} status={w['status']} " - f"faults={len(w.get('faults') or [])}" + f" {wid} owner={rec['owner']} provider={rec['provider']} " + f"status={rec['status']} faults={rec['faults']}" ) print(" --- base worker status ---") - print(" " + cli.replace("\n", "\n ").strip()) - cli_has_all = all(wid in cli for wid in api_ids) - print(f" ids in both surfaces: {cli_has_all}") + print(" " + cli_text.replace("\n", "\n ").strip()) + if not agree: + only_api = {k: api_view[k] for k in api_view.keys() - cli_view.keys()} + only_cli = {k: cli_view[k] for k in cli_view.keys() - api_view.keys()} + mismatched = { + k: {"api": api_view[k], "cli": cli_view[k]} + for k in api_view.keys() & cli_view.keys() + if api_view[k] != cli_view[k] + } + print(f" DIFF only_api={only_api} only_cli={only_cli} mismatched={mismatched}") + else: + print(f" API and CLI fleet views identical for {len(api_view)} worker(s)") for w in workers: h.kill(w) for owner in ("alice", "bob"): _wait_stale(h, ss58(MINERS[owner])) - print(f" RESULT: {'PASS' if cli_has_all and api_ids else 'FAIL'}") - return bool(cli_has_all and api_ids) + print(f" RESULT: {'PASS' if agree else 'FAIL'}") + return agree + + +#: Worker lifecycle statuses rendered by ``base worker status`` (used to reliably +#: distinguish fleet rows from any interleaved log lines when parsing the CLI). +_WORKER_STATUSES = {"active", "stale", "pending", "retired"} + + +def _fleet_from_api(api_workers: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Project GET /v1/workers into a comparable {worker_id: fields} map.""" + + return { + w["worker_id"]: { + "owner": w["miner_hotkey"], + "provider": w["provider"], + "status": w["status"], + "faults": len(w.get("faults") or []), + } + for w in api_workers + } + + +def _parse_cli_workers(cli: str) -> dict[str, dict[str, Any]]: + """Parse `base worker status` table rows into a comparable map. + + Each fleet row is ``WORKER_ID OWNER PROVIDER STATUS FAULTS LAST_SEEN`` (all + single whitespace-free tokens). Rows are identified by a known lifecycle + status + numeric fault column so interleaved log lines are ignored. + """ + + rows: dict[str, dict[str, Any]] = {} + for line in cli.splitlines(): + parts = line.split() + if len(parts) < 6: + continue + worker_id, owner, provider, status, faults = parts[:5] + if status not in _WORKER_STATUSES or not faults.isdigit(): + continue + rows[worker_id] = { + "owner": owner, + "provider": provider, + "status": status, + "faults": int(faults), + } + return rows def _completed_with_score(sub: dict[str, Any]) -> dict[str, Any] | None: @@ -668,13 +767,6 @@ def _score_of(sub: dict[str, Any]) -> Any: return None -def _faults_for_worker(workers: list[dict[str, Any]], worker_id: str) -> list[dict[str, Any]]: - for w in workers: - if w["worker_id"] == worker_id: - return w.get("faults") or [] - return [] - - def _wait_stale(h: Harness, owner_hotkey: str) -> None: time.sleep(WORKER_TTL + 2) diff --git a/scripts/mission/legacy_smoke.py b/scripts/mission/legacy_smoke.py index fba5482..a5d68ea 100644 --- a/scripts/mission/legacy_smoke.py +++ b/scripts/mission/legacy_smoke.py @@ -38,6 +38,7 @@ import sqlite3 import sys import time +import uuid from pathlib import Path from typing import Any @@ -211,7 +212,9 @@ def run(workdir: Path) -> bool: time.sleep(5) # let it register + heartbeat before work exists print("\n=== (1) admission OFF: submission accepted with no NO_ACTIVE_WORKER 403 ===") - resp = h.submit(nonce="legacy-1") + # Fresh nonce per run so re-running against a reused --workdir (persistent + # sqlite nonce store) is not rejected as a replay before routing. + resp = h.submit(nonce=f"legacy-{uuid.uuid4().hex}") print(f" POST /v1/submissions -> HTTP {resp.status_code} body={resp.text[:160]}") ok_accept = resp.status_code < 300 and "NO_ACTIVE_WORKER" not in resp.text results["(1) submission accepted, no admission 403"] = ok_accept diff --git a/scripts/mission/mission_prism.py b/scripts/mission/mission_prism.py index d0b62ca..7b2b58a 100644 --- a/scripts/mission/mission_prism.py +++ b/scripts/mission/mission_prism.py @@ -16,7 +16,9 @@ from __future__ import annotations +import atexit import json +import logging import sys from pathlib import Path from typing import Any @@ -73,14 +75,49 @@ def build_settings(config: dict[str, Any]) -> PrismSettings: ) +def _flush_streams() -> None: + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except (ValueError, OSError): + pass + + +def _configure_harness_logging(level: int = logging.INFO) -> None: + """Line-buffer stdout/stderr and route logs there so drill logs are + inspectable after teardown. + + The harness redirects each spawned process' stdout/stderr to a log file and + tears it down with SIGTERM; block-buffered output would be lost on kill, + leaving a 0-byte log. Line-buffering flushes every completed log line + immediately, and an ``atexit`` flush covers the graceful-shutdown path. + """ + + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + try: + reconfigure(line_buffering=True) + except (ValueError, OSError): + pass + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + atexit.register(_flush_streams) + + def main() -> None: + _configure_harness_logging() config = _load_config() settings = build_settings(config) uvicorn.run( create_app(settings), host=str(config.get("host", "127.0.0.1")), port=int(config["port"]), - log_level=str(config.get("log_level", "warning")), + log_level=str(config.get("log_level", "info")), ) diff --git a/scripts/mission/mission_worker.py b/scripts/mission/mission_worker.py index 2587955..f67a1e0 100644 --- a/scripts/mission/mission_worker.py +++ b/scripts/mission/mission_worker.py @@ -16,7 +16,9 @@ from __future__ import annotations import asyncio +import atexit import json +import logging import signal import sys from pathlib import Path @@ -161,7 +163,42 @@ async def _run(config: dict[str, Any]) -> None: await agent.run_forever(shutdown) +def _flush_streams() -> None: + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except (ValueError, OSError): + pass + + +def _configure_harness_logging(level: int = logging.INFO) -> None: + """Line-buffer stdout/stderr and route logs there so drill logs are + inspectable after teardown. + + The harness redirects each spawned process' stdout/stderr to a log file and + tears it down with SIGTERM; block-buffered output would be lost on kill, + leaving a 0-byte log. Line-buffering flushes every completed log line + immediately, and an ``atexit`` flush covers the graceful-shutdown path. + """ + + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + try: + reconfigure(line_buffering=True) + except (ValueError, OSError): + pass + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + atexit.register(_flush_streams) + + def main() -> None: + _configure_harness_logging() asyncio.run(_run(_load_config()))