From fe47c2179c5dbca4b332c009c791e234d578764a Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Wed, 8 Jul 2026 01:32:29 +0200 Subject: [PATCH] FE-1158: Add boolean predicates to experiment metrics Predicates are boolean-returning custom-code metrics evaluated per run: each run is tested every frame until the predicate first returns true, the time is recorded, and the run is not tested again. - Compile predicates through the HIR metric surface with a boolean return type (HirCompileOptions.metricReturnTypes), sharing the metric buffer ABI and LSP session validation. - Model predicate specs as a discriminated union member (MonteCarloPredicateSpec) travelling in-band in metricSpecs; runtimes split once via splitMonteCarloMetricSpecs. - Stream versioned predicate snapshots from the worker into a new predicates store on the experiment handle. - UI: Predicate toggle on custom-code experiment metrics; view-drawer boxes show floored true-percentage, run counts, and the median simulation time at which runs first satisfied the predicate. - Cover the compiled boolean artifact path end-to-end against real frame buffers, and update the user docs and BUFFER_ABI.md. --- libs/@hashintel/petrinaut-core/src/hir.ts | 2 + .../petrinaut-core/src/hir/BUFFER_ABI.md | 10 +- .../petrinaut-core/src/hir/compile.ts | 17 +- .../petrinaut-core/src/hir/emit-buffer-js.ts | 6 +- .../petrinaut-core/src/hir/instantiate.ts | 16 +- .../petrinaut-core/src/hir/lint.test.ts | 8 + .../petrinaut-core/src/hir/surface-context.ts | 7 + .../petrinaut-core/src/hir/typecheck.ts | 11 + libs/@hashintel/petrinaut-core/src/index.ts | 10 + .../petrinaut-core/src/lsp/language-client.ts | 6 +- .../lib/create-sdcpn-language-service.test.ts | 22 ++ .../src/lsp/lib/generate-virtual-files.ts | 8 +- .../src/lsp/worker/language-server.worker.ts | 12 +- .../petrinaut-core/src/lsp/worker/protocol.ts | 5 +- .../src/simulation/frames/hir-metric.ts | 80 +++++-- .../petrinaut-core/src/simulation/index.ts | 11 + .../src/simulation/monte-carlo/index.ts | 11 + .../simulation/monte-carlo/metrics/index.ts | 17 +- .../monte-carlo/metrics/predicates.ts | 149 +++++++++++++ .../simulation/monte-carlo/metrics/specs.ts | 32 ++- .../simulation/monte-carlo/metrics/types.ts | 89 +++++++- .../monte-carlo/runtime/experiment.test.ts | 201 +++++++++++++++++- .../monte-carlo/runtime/experiment.ts | 93 +++++++- .../simulation/monte-carlo/worker/messages.ts | 8 + .../monte-carlo/worker/monte-carlo.worker.ts | 52 ++++- .../petrinaut-core/src/workers/monte-carlo.ts | 1 + libs/@hashintel/petrinaut/docs/experiments.md | 12 ++ .../src/react/experiments/context.ts | 21 +- .../src/react/experiments/provider.test.tsx | 87 +++++++- .../src/react/experiments/provider.tsx | 76 ++++--- .../petrinaut/src/react/lsp/context.ts | 2 + .../experiments/create-experiment-drawer.tsx | 129 ++++++++++- .../experiment-metric-lsp-validation.ts | 9 +- .../experiments-story-fixtures.tsx | 2 + .../experiments/view-experiment-drawer.tsx | 85 +++++++- .../SimulateView/metrics/metric-form.tsx | 11 +- 36 files changed, 1215 insertions(+), 103 deletions(-) create mode 100644 libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/predicates.ts diff --git a/libs/@hashintel/petrinaut-core/src/hir.ts b/libs/@hashintel/petrinaut-core/src/hir.ts index c6d026dcdba..c4b65d3b22e 100644 --- a/libs/@hashintel/petrinaut-core/src/hir.ts +++ b/libs/@hashintel/petrinaut-core/src/hir.ts @@ -20,8 +20,10 @@ export { } from "./hir/analyze"; export { compileHirArtifacts, + type HirCompileOptions, type HirCompileFailure, type HirCompileResult, + type HirMetricReturnType, } from "./hir/compile"; export { hirDistributionRuntime, diff --git a/libs/@hashintel/petrinaut-core/src/hir/BUFFER_ABI.md b/libs/@hashintel/petrinaut-core/src/hir/BUFFER_ABI.md index 83f87d39a8a..eff6ce1cab6 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/BUFFER_ABI.md +++ b/libs/@hashintel/petrinaut-core/src/hir/BUFFER_ABI.md @@ -101,12 +101,18 @@ The field order matches `TokenSlotLayout.realFieldF64Offsets`. ## Metrics -Metrics read a frame's raw token region and dense place metadata: +Metric-shaped frame readers — numeric metrics and boolean experiment +predicates — read a frame's raw token region and dense place metadata: ```ts -(f64, u64, u8, placeCounts, placeOffsets) => number; +(f64, u64, u8, placeCounts, placeOffsets) => number | boolean; ``` +Numeric metrics must return a finite number. Predicates are compiled with a +boolean return type (`HirMetricContext.returnType` / +`HirCompileOptions.metricReturnTypes`) and must return a boolean; the runtime +evaluators enforce the respective return type per call. + `HirMetricArtifact.placeNames` records places in first-reference order. At instantiation, `__places[ordinal]` maps those names to frame place indexes. diff --git a/libs/@hashintel/petrinaut-core/src/hir/compile.ts b/libs/@hashintel/petrinaut-core/src/hir/compile.ts index 1dd414ef199..73610e5311f 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/compile.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/compile.ts @@ -57,6 +57,17 @@ export type HirCompileResult = { failures: HirCompileFailure[]; }; +export type HirMetricReturnType = "number" | "boolean"; + +export type HirCompileOptions = { + /** + * Optional per-metric return-type override. Regular model metrics default to + * numeric; experiment predicates use the same metric-shaped frame surface but + * compile as boolean-returning artifacts. + */ + metricReturnTypes?: Readonly>; +}; + function notCompilableDiagnostic(fn: HirFunction): HirDiagnostic { return { code: "hir:not-compilable", @@ -102,6 +113,7 @@ function lowerAndCheck( export function compileHirArtifacts( sdcpn: SDCPN, extensions: PetrinautExtensionSettings = DEFAULT_PETRINAUT_EXTENSIONS, + options: HirCompileOptions = {}, ): HirCompileResult { const sanitized = sanitizeSDCPNForExtensions(sdcpn, extensions); const artifacts: HirArtifacts = { @@ -248,7 +260,10 @@ export function compileHirArtifacts( // error, but they must not block the rest of the net from compiling. continue; } - const item = lowerAndCheck(metric.code, "metric", metricContext); + const item = lowerAndCheck(metric.code, "metric", { + ...metricContext, + returnType: options.metricReturnTypes?.[metric.id] ?? "number", + }); if (!item.ok) { failures.push({ itemId: metric.id, diff --git a/libs/@hashintel/petrinaut-core/src/hir/emit-buffer-js.ts b/libs/@hashintel/petrinaut-core/src/hir/emit-buffer-js.ts index 73c0a5907d5..b987b3d9843 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/emit-buffer-js.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/emit-buffer-js.ts @@ -7,7 +7,7 @@ * * lambda: (f64, u64, u8, placeBases, indices) => number | boolean * kernel: (f64, u64, u8, placeBases, indices, outF64, outU64, outU8, sink) => void - * metric: (f64, u64, u8, placeCounts, placeOffsets) => number + * metric-shaped frame reader: (f64, u64, u8, placeCounts, placeOffsets) => number | boolean * * The full ABI is documented in `BUFFER_ABI.md`. * @@ -928,9 +928,9 @@ export function emitBufferKernelJs( } /** - * Emits a buffer-ABI metric program (token format v2): + * Emits a buffer-ABI metric-shaped frame-reader program (token format v2): * - * (f64, u64, u8, placeCounts, placeOffsets) => number + * (f64, u64, u8, placeCounts, placeOffsets) => number | boolean * * - `f64`/`u64`/`u8` — shared views over the frame's token byte region. * - `placeCounts`/`placeOffsets` — the frame's dense per-place token counts diff --git a/libs/@hashintel/petrinaut-core/src/hir/instantiate.ts b/libs/@hashintel/petrinaut-core/src/hir/instantiate.ts index 5d1e103875c..1abbe0c7820 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/instantiate.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/instantiate.ts @@ -57,8 +57,9 @@ export type HirCompiledBufferLambda = ( /** * Buffer-ABI metric: reads a frame's packed token region through shared * views plus its dense per-place `placeCounts`/`placeOffsets` arrays and - * returns one number. The place-ordinal→frame-place-index table and string - * pool are pre-bound at instantiation. + * returns one scalar. Numeric metrics and boolean predicates share this ABI. + * The place-ordinal→frame-place-index table and string pool are pre-bound at + * instantiation. */ export type HirCompiledMetric = ( f64: Float64Array, @@ -66,7 +67,7 @@ export type HirCompiledMetric = ( u8: Uint8Array, placeCounts: Uint32Array, placeOffsets: Uint32Array, -) => number; +) => number | boolean; /** Buffer-ABI kernel: writes output attributes into per-transition staging * (place-major, baked offsets); RNG-consuming slots defer through the sink. */ @@ -203,11 +204,12 @@ export function instantiateHirBufferKernel( } /** - * Instantiates a compiled metric program. `placeIndices[ordinal]` maps each + * Instantiates a compiled metric-shaped frame-reader program. + * `placeIndices[ordinal]` maps each * of the artifact's `placeNames` to the frame's place index (resolve once - * per experiment/simulation, not per frame). Metrics have no parameters - * object and never intern strings; `__params`/`__dist` are bound for ABI - * parity with the other programs but unused. + * per experiment/simulation, not per frame). Metrics and predicates have no + * parameters object and never intern strings; `__params`/`__dist` are bound + * for ABI parity with the other programs but unused. */ export function instantiateHirMetric( source: string, diff --git a/libs/@hashintel/petrinaut-core/src/hir/lint.test.ts b/libs/@hashintel/petrinaut-core/src/hir/lint.test.ts index a4ce4337038..0d1615d2091 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/lint.test.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/lint.test.ts @@ -331,6 +331,14 @@ return all.reduce((sum, t) => sum + t.x, 0) / all.length;`, "error", ]); + // Predicate sessions reuse the metric state surface but return booleans. + expect( + codes(`return true;`, { ...metricContext, returnType: "boolean" }), + ).toEqual([]); + expect( + codes(`return 1;`, { ...metricContext, returnType: "boolean" }), + ).toContainEqual(["hir:metric-return", "error"]); + // Unknown place. expect( codes(`return state.places.Missing.count;`, metricContext), diff --git a/libs/@hashintel/petrinaut-core/src/hir/surface-context.ts b/libs/@hashintel/petrinaut-core/src/hir/surface-context.ts index 3371e0b0917..f4ca6685906 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/surface-context.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/surface-context.ts @@ -126,6 +126,13 @@ export type HirMetricContext = { surface: "metric"; /** Metrics have no parameters object — always empty. */ parameters: HirParameterInfo[]; + /** + * The scalar result expected from the metric-shaped frame reader surface. + * + * Experiment predicates reuse the same `state` object and buffer emission as + * metrics, but they return a boolean instead of a numeric sample. + */ + returnType?: "number" | "boolean"; /** ALL places of the root net, keyed by display name (last name wins for * duplicates, matching the runtime object-key overwrite). */ places: HirMetricPlaceInfo[]; diff --git a/libs/@hashintel/petrinaut-core/src/hir/typecheck.ts b/libs/@hashintel/petrinaut-core/src/hir/typecheck.ts index bdc2f502a9e..6fbc0d46e44 100644 --- a/libs/@hashintel/petrinaut-core/src/hir/typecheck.ts +++ b/libs/@hashintel/petrinaut-core/src/hir/typecheck.ts @@ -747,6 +747,17 @@ class Typechecker { return; } case "metric": { + if (context.returnType === "boolean") { + if (!isBoolish(returnType)) { + this.report( + bodySpan, + "hir:metric-return", + `Predicates must return a boolean, got ${formatHirType(returnType)}.`, + ); + } + return; + } + if (!isNumeric(returnType)) { this.report( bodySpan, diff --git a/libs/@hashintel/petrinaut-core/src/index.ts b/libs/@hashintel/petrinaut-core/src/index.ts index 79377fe7d18..4308e08193e 100644 --- a/libs/@hashintel/petrinaut-core/src/index.ts +++ b/libs/@hashintel/petrinaut-core/src/index.ts @@ -181,8 +181,10 @@ export type { MonteCarloExperiment, MonteCarloExperimentEvent, MonteCarloExperimentMetrics, + MonteCarloExperimentPredicates, MonteCarloExperimentState, MonteCarloExpressionMetricSpec, + MonteCarloNumericMetricSpec, MonteCarloFrameMetric, MonteCarloFrameMetricContext, MonteCarloMetricDistributionBinning, @@ -191,10 +193,12 @@ export type { MonteCarloMetricNumericAccumulatorState, MonteCarloMetricSpec, MonteCarloMetricSpecBase, + MonteCarloMetricType, MonteCarloMetricRunOutput, MonteCarloMetricRunStatus, MonteCarloMetricValueAccumulator, MonteCarloPlaceTokenCountMeanMetricSpec, + MonteCarloPredicateSpec, MonteCarloRunFrameMetricView, MonteCarloRunFrameMetricVisitor, MonteCarloRunConfig, @@ -214,6 +218,11 @@ export type { MonteCarloUserDefinedMetricSampleRuns, MonteCarloUserDefinedScalarMetricFrame, MonteCarloUserDefinedMetricTimeAggregation, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateMeasureInput, + MonteCarloUserDefinedPredicateRunResult, + MonteCarloUserDefinedPredicateSnapshot, MonteCarloTransitionFiringCountMetricSpec, MonteCarloWorkerProgress, } from "./simulation"; @@ -248,6 +257,7 @@ export type { // LSP worker, runtime instantiation in ./hir-runtime) --- export type { HirArtifacts, + HirCompileOptions, HirCompileFailure, HirCompileResult, HirDiagnostic, diff --git a/libs/@hashintel/petrinaut-core/src/lsp/language-client.ts b/libs/@hashintel/petrinaut-core/src/lsp/language-client.ts index b939b384a29..01cf0934cb0 100644 --- a/libs/@hashintel/petrinaut-core/src/lsp/language-client.ts +++ b/libs/@hashintel/petrinaut-core/src/lsp/language-client.ts @@ -8,7 +8,7 @@ import { import type { PetrinautExtensionSettings } from "../extensions"; // Type-only: must not pull the compiler (`typescript`) into client bundles. -import type { HirCompileResult } from "../hir"; +import type { HirCompileOptions, HirCompileResult } from "../hir"; import type { ReadableStore } from "../store"; import type { SDCPN } from "../types/sdcpn"; import type { @@ -93,6 +93,7 @@ export interface LanguageClient { this: void, sdcpn: SDCPN, extensions?: PetrinautExtensionSettings, + options?: HirCompileOptions, ): Promise; /** @@ -314,10 +315,11 @@ export function createLanguageClient( position, }); }, - requestHirArtifacts(sdcpn, extensions) { + requestHirArtifacts(sdcpn, extensions, options) { return sendRequest("sdcpn/compileHirArtifacts", { sdcpn, extensions, + options, }); }, diff --git a/libs/@hashintel/petrinaut-core/src/lsp/lib/create-sdcpn-language-service.test.ts b/libs/@hashintel/petrinaut-core/src/lsp/lib/create-sdcpn-language-service.test.ts index 0bfabc4f17e..780f81634ca 100644 --- a/libs/@hashintel/petrinaut-core/src/lsp/lib/create-sdcpn-language-service.test.ts +++ b/libs/@hashintel/petrinaut-core/src/lsp/lib/create-sdcpn-language-service.test.ts @@ -409,4 +409,26 @@ describe("SDCPNLanguageServer completions", () => { expect(names).toContain("gravity"); }); }); + + describe("metric session files", () => { + it("wraps boolean-return sessions as predicates", () => { + const sdcpn = createSDCPN({ + places: [{ id: "place1", name: "Done", colorId: null }], + }); + const server = new SDCPNLanguageServer(); + server.syncMetricFiles(sdcpn, { + sessionId: "predicate-session", + code: "return state.places.Done.count > 0;", + returnType: "boolean", + }); + + const filePath = getItemFilePath("metric-code", { + sessionId: "predicate-session", + }); + + expect(server.getFileContent(filePath)).toContain( + "function __metric(state: MetricState): boolean", + ); + }); + }); }); diff --git a/libs/@hashintel/petrinaut-core/src/lsp/lib/generate-virtual-files.ts b/libs/@hashintel/petrinaut-core/src/lsp/lib/generate-virtual-files.ts index 816e3dfc655..964595f1122 100644 --- a/libs/@hashintel/petrinaut-core/src/lsp/lib/generate-virtual-files.ts +++ b/libs/@hashintel/petrinaut-core/src/lsp/lib/generate-virtual-files.ts @@ -583,8 +583,9 @@ export function generateScenarioSessionFiles( */ export type MetricSessionData = { sessionId: string; - /** Metric function body (must `return` a finite number). */ + /** Metric-shaped function body. Metrics return numbers; predicates return booleans. */ code: string; + returnType?: "number" | "boolean"; }; /** @@ -592,7 +593,8 @@ export type MetricSessionData = { * * The user code is a function body whose `state` parameter exposes * `state.places.` with `count` and (for colored places) `tokens`. - * The expression is wrapped so the body is type-checked as `(state) => number`. + * The expression is wrapped so the body is type-checked as `(state) => number` + * for metrics and `(state) => boolean` for predicates. */ export function generateMetricSessionFiles( sdcpn: SDCPN, @@ -659,7 +661,7 @@ export function generateMetricSessionFiles( files.set(codePath, { prefix: [ `import type { MetricState } from "${defsPath}";`, - `function __metric(state: MetricState): number {`, + `function __metric(state: MetricState): ${session.returnType ?? "number"} {`, "", ].join("\n"), content: session.code, diff --git a/libs/@hashintel/petrinaut-core/src/lsp/worker/language-server.worker.ts b/libs/@hashintel/petrinaut-core/src/lsp/worker/language-server.worker.ts index 4d95f74d5c9..5039bca0b4d 100644 --- a/libs/@hashintel/petrinaut-core/src/lsp/worker/language-server.worker.ts +++ b/libs/@hashintel/petrinaut-core/src/lsp/worker/language-server.worker.ts @@ -147,7 +147,10 @@ function publishAllDiagnostics( ); if (!hasTsError) { allDiags.push( - ...getHirDiagnosticsForItem(userContent, metricHirContext), + ...getHirDiagnosticsForItem(userContent, { + ...metricHirContext, + returnType: session.returnType ?? "number", + }), ); } params.push({ @@ -197,6 +200,7 @@ function toMetricSessionData(params: MetricSessionParams): MetricSessionData { return { sessionId: params.sessionId, code: params.code, + returnType: params.returnType, }; } @@ -384,7 +388,11 @@ workerRuntime.onMessage((data) => { const { id } = data; respond( id, - compileHirArtifacts(data.params.sdcpn, data.params.extensions), + compileHirArtifacts( + data.params.sdcpn, + data.params.extensions, + data.params.options, + ), ); break; } diff --git a/libs/@hashintel/petrinaut-core/src/lsp/worker/protocol.ts b/libs/@hashintel/petrinaut-core/src/lsp/worker/protocol.ts index 9df7c2aee59..3f59d8ae273 100644 --- a/libs/@hashintel/petrinaut-core/src/lsp/worker/protocol.ts +++ b/libs/@hashintel/petrinaut-core/src/lsp/worker/protocol.ts @@ -13,6 +13,7 @@ * upstream package directly. */ import type { PetrinautExtensionSettings } from "../../extensions"; +import type { HirCompileOptions } from "../../hir"; import type { SDCPN, ScenarioParameter } from "../../types/sdcpn"; import type { Diagnostic, @@ -51,8 +52,9 @@ export type ScenarioSessionParams = { */ export type MetricSessionParams = { sessionId: string; - /** Metric function body (must `return` a finite number). */ + /** Metric-shaped function body. Metrics return numbers; predicates return booleans. */ code: string; + returnType?: "number" | "boolean"; }; /** Position in a text document (LSP standard: line/character based). */ @@ -143,6 +145,7 @@ type ClientRequest = params: { sdcpn: SDCPN; extensions?: PetrinautExtensionSettings; + options?: HirCompileOptions; }; }; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/frames/hir-metric.ts b/libs/@hashintel/petrinaut-core/src/simulation/frames/hir-metric.ts index 046244a2863..fb0c8c65a77 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/frames/hir-metric.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/frames/hir-metric.ts @@ -8,24 +8,23 @@ import type { Place } from "../../types/sdcpn"; import type { SimulationFrameReader } from "../api"; /** - * Binds a compiled HIR metric artifact to frame readers. + * Binds a compiled HIR metric-shaped artifact to frame readers. * * The returned evaluator resolves the artifact's referenced place names to * frame place indices once (on the first frame — the layout is stable per * simulation/experiment), instantiates the program once, and then runs it - * against each frame's raw buffer view. Throws (with the metric label) when - * a referenced place does not exist, when the frame source exposes no raw - * buffer access, or when the program returns a non-finite value — mirroring - * the legacy sandboxed-metric error semantics. + * against each frame's raw buffer view. Throws when a referenced place does + * not exist or when the frame source exposes no raw buffer access. */ -export function createHirMetricEvaluator(args: { +function createHirFrameEvaluator(args: { /** Display name used in error messages (metric label or name). */ - metricName: string; + itemName: string; + itemKind: "Metric" | "Predicate"; artifact: HirMetricArtifact; /** Root-net places, used to resolve place display names to ids. */ places: readonly Pick[]; -}): (frame: SimulationFrameReader) => number { - const { metricName, artifact, places } = args; +}): (frame: SimulationFrameReader) => number | boolean { + const { itemName, itemKind, artifact, places } = args; // Last place wins for duplicate names, matching the HIR metric context. const placeIdByName = new Map(places.map((place) => [place.name, place.id])); @@ -41,7 +40,7 @@ export function createHirMetricEvaluator(args: { const raw = frame.getRawView?.(); if (!raw) { throw new Error( - `Metric "${metricName}" cannot run here — this frame source does not expose raw buffer access.`, + `${itemKind} "${itemName}" cannot run here — this frame source does not expose raw buffer access.`, ); } @@ -53,7 +52,7 @@ export function createHirMetricEvaluator(args: { placeId === undefined ? undefined : raw.placeIndexById.get(placeId); if (placeIndex === undefined) { throw new Error( - `Metric "${metricName}" reads place "${placeName}", which does not exist in this simulation.`, + `${itemKind} "${itemName}" reads place "${placeName}", which does not exist in this simulation.`, ); } placeIndices[ordinal] = placeIndex; @@ -74,9 +73,66 @@ export function createHirMetricEvaluator(args: { raw.placeOffsets, ); currentPool = null; + return result; + }; +} + +/** + * Binds a compiled HIR metric artifact to frame readers. + * + * The returned evaluator validates that the program returns a finite number, + * mirroring the legacy sandboxed-metric error semantics. + */ +export function createHirMetricEvaluator(args: { + /** Display name used in error messages (metric label or name). */ + metricName: string; + artifact: HirMetricArtifact; + /** Root-net places, used to resolve place display names to ids. */ + places: readonly Pick[]; +}): (frame: SimulationFrameReader) => number { + const evaluate = createHirFrameEvaluator({ + itemName: args.metricName, + itemKind: "Metric", + artifact: args.artifact, + places: args.places, + }); + + return (frame) => { + const result = evaluate(frame); if (typeof result !== "number" || !Number.isFinite(result)) { throw new Error( - `Metric "${metricName}" returned ${String(result)}, expected a finite number.`, + `Metric "${args.metricName}" returned ${String(result)}, expected a finite number.`, + ); + } + return result; + }; +} + +/** + * Binds a compiled HIR predicate artifact to frame readers. + * + * Predicates intentionally share the metric `state` surface and buffer ABI, + * but the compiled program must return a boolean. + */ +export function createHirPredicateEvaluator(args: { + /** Display name used in error messages (predicate label). */ + predicateName: string; + artifact: HirMetricArtifact; + /** Root-net places, used to resolve place display names to ids. */ + places: readonly Pick[]; +}): (frame: SimulationFrameReader) => boolean { + const evaluate = createHirFrameEvaluator({ + itemName: args.predicateName, + itemKind: "Predicate", + artifact: args.artifact, + places: args.places, + }); + + return (frame) => { + const result = evaluate(frame); + if (typeof result !== "boolean") { + throw new Error( + `Predicate "${args.predicateName}" returned ${String(result)}, expected a boolean.`, ); } return result; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/index.ts b/libs/@hashintel/petrinaut-core/src/simulation/index.ts index 71f1cd82472..b6b1a5b893b 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/index.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/index.ts @@ -25,6 +25,8 @@ export { createMonteCarloSimulator, createMonteCarloUserDefinedMetricConfigsFromSpecs, createMonteCarloUserDefinedMetric, + createMonteCarloUserDefinedPredicate, + createMonteCarloUserDefinedPredicateConfigsFromSpecs, } from "./monte-carlo"; export type { CreateMonteCarloExperimentConfig, @@ -33,8 +35,10 @@ export type { MonteCarloExperiment, MonteCarloExperimentEvent, MonteCarloExperimentMetrics, + MonteCarloExperimentPredicates, MonteCarloExperimentState, MonteCarloExpressionMetricSpec, + MonteCarloNumericMetricSpec, MonteCarloFrameMetric, MonteCarloFrameMetricContext, MonteCarloMetricDistributionBinning, @@ -43,10 +47,12 @@ export type { MonteCarloMetricNumericAccumulatorState, MonteCarloMetricSpec, MonteCarloMetricSpecBase, + MonteCarloMetricType, MonteCarloMetricRunOutput, MonteCarloMetricRunStatus, MonteCarloMetricValueAccumulator, MonteCarloPlaceTokenCountMeanMetricSpec, + MonteCarloPredicateSpec, MonteCarloRunFrameMetricView, MonteCarloRunFrameMetricVisitor, MonteCarloRunConfig, @@ -66,6 +72,11 @@ export type { MonteCarloUserDefinedMetricSampleRuns, MonteCarloUserDefinedScalarMetricFrame, MonteCarloUserDefinedMetricTimeAggregation, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateMeasureInput, + MonteCarloUserDefinedPredicateRunResult, + MonteCarloUserDefinedPredicateSnapshot, MonteCarloTransitionFiringCountMetricSpec, MonteCarloWorkerProgress, } from "./monte-carlo"; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/index.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/index.ts index 3e27827423f..f3c8af221b0 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/index.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/index.ts @@ -5,6 +5,8 @@ export { createMonteCarloMetricNumericAccumulator, createMonteCarloUserDefinedMetricConfigsFromSpecs, createMonteCarloUserDefinedMetric, + createMonteCarloUserDefinedPredicate, + createMonteCarloUserDefinedPredicateConfigsFromSpecs, } from "./metrics"; export { createMonteCarloExperiment } from "./runtime/experiment"; export type { @@ -20,6 +22,7 @@ export type { export type { MonteCarloActiveRunPlaceCountsVisitor, MonteCarloExpressionMetricSpec, + MonteCarloNumericMetricSpec, MonteCarloFrameMetric, MonteCarloFrameMetricContext, MonteCarloMetricDistributionBinning, @@ -28,10 +31,12 @@ export type { MonteCarloMetricNumericAccumulatorState, MonteCarloMetricSpec, MonteCarloMetricSpecBase, + MonteCarloMetricType, MonteCarloMetricRunOutput, MonteCarloMetricRunStatus, MonteCarloMetricValueAccumulator, MonteCarloPlaceTokenCountMeanMetricSpec, + MonteCarloPredicateSpec, MonteCarloRunFrameMetricView, MonteCarloRunFrameMetricVisitor, MonteCarloTransitionFiringCountMetricSpec, @@ -45,12 +50,18 @@ export type { MonteCarloUserDefinedMetricSampleRuns, MonteCarloUserDefinedScalarMetricFrame, MonteCarloUserDefinedMetricTimeAggregation, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateMeasureInput, + MonteCarloUserDefinedPredicateRunResult, + MonteCarloUserDefinedPredicateSnapshot, } from "./metrics"; export type { CreateMonteCarloExperimentConfig, MonteCarloExperiment, MonteCarloExperimentEvent, MonteCarloExperimentMetrics, + MonteCarloExperimentPredicates, MonteCarloExperimentState, } from "./runtime/experiment"; export type { MonteCarloWorkerProgress } from "./worker/messages"; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/index.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/index.ts index adbe17cd8ba..e12fb344eee 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/index.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/index.ts @@ -3,7 +3,14 @@ export { createMonteCarloMetricHistogramAccumulator, createMonteCarloMetricNumericAccumulator, } from "./accumulators"; -export { createMonteCarloUserDefinedMetricConfigsFromSpecs } from "./specs"; +export { + createMonteCarloUserDefinedMetricConfigsFromSpecs, + splitMonteCarloMetricSpecs, +} from "./specs"; +export { + createMonteCarloUserDefinedPredicate, + createMonteCarloUserDefinedPredicateConfigsFromSpecs, +} from "./predicates"; export { createMonteCarloUserDefinedMetric } from "./user-defined"; export type { MonteCarloMetricHistogramAccumulatorState, @@ -19,6 +26,9 @@ export type { MonteCarloFrameMetricContext, MonteCarloMetricSpec, MonteCarloMetricSpecBase, + MonteCarloMetricType, + MonteCarloNumericMetricSpec, + MonteCarloPredicateSpec, MonteCarloMetricRunOutput, MonteCarloMetricRunStatus, MonteCarloPlaceTokenCountMeanMetricSpec, @@ -35,4 +45,9 @@ export type { MonteCarloUserDefinedMetricSampleRuns, MonteCarloUserDefinedScalarMetricFrame, MonteCarloUserDefinedMetricTimeAggregation, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateMeasureInput, + MonteCarloUserDefinedPredicateRunResult, + MonteCarloUserDefinedPredicateSnapshot, } from "./types"; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/predicates.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/predicates.ts new file mode 100644 index 00000000000..5497c73365b --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/predicates.ts @@ -0,0 +1,149 @@ +import { createHirPredicateEvaluator } from "../../frames/hir-metric"; + +import type { SDCPN } from "../../../types/sdcpn"; +import type { + MonteCarloMetricRunStatus, + MonteCarloPredicateSpec, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateRunResult, + MonteCarloUserDefinedPredicateSnapshot, +} from "./types"; + +/** Errored runs are never tested — they stay false in the results. */ +function shouldTestPredicateRun(status: MonteCarloMetricRunStatus): boolean { + return status !== "error"; +} + +function cloneResults( + results: readonly MonteCarloUserDefinedPredicateRunResult[], +): MonteCarloUserDefinedPredicateRunResult[] { + return results.map((result) => ({ ...result })); +} + +function createSnapshot(args: { + id: string; + label: string; + frameNumber: number; + time: number; + results: readonly MonteCarloUserDefinedPredicateRunResult[]; +}): MonteCarloUserDefinedPredicateSnapshot { + return { + predicateId: args.id, + label: args.label, + frameNumber: args.frameNumber, + time: args.time, + runCount: args.results.length, + trueRunCount: args.results.filter((result) => result.value).length, + runResults: cloneResults(args.results), + }; +} + +function createExpressionPredicateConfig( + spec: MonteCarloPredicateSpec, + sdcpn: SDCPN, +): MonteCarloUserDefinedPredicateConfig { + // Expression predicates run exclusively as HIR-compiled buffer programs. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- guards specs built before artifact threading (e.g. persisted configs) + if (!spec.artifact) { + throw new Error( + `Predicate "${spec.label}" has no compiled artifact — expression predicates must be compiled through the HIR before starting an experiment.`, + ); + } + + const evaluate = createHirPredicateEvaluator({ + predicateName: spec.label, + artifact: spec.artifact, + places: sdcpn.places, + }); + + return { + id: spec.id, + label: spec.label, + test: ({ frame }) => evaluate(frame), + }; +} + +export function createMonteCarloUserDefinedPredicateConfigsFromSpecs( + specs: readonly MonteCarloPredicateSpec[], + sdcpn: SDCPN, +): MonteCarloUserDefinedPredicateConfig[] { + return specs.map((spec) => createExpressionPredicateConfig(spec, sdcpn)); +} + +export function createMonteCarloUserDefinedPredicate( + config: MonteCarloUserDefinedPredicateConfig, +): MonteCarloUserDefinedPredicate { + const label = config.label ?? config.id; + let results: MonteCarloUserDefinedPredicateRunResult[] = []; + let latestSnapshot: MonteCarloUserDefinedPredicateSnapshot | null = null; + let version = 0; + + const ensureResults = (runCount: number) => { + if (results.length === runCount) { + return; + } + + results = Array.from({ length: runCount }, (_, runIndex) => ({ + runIndex, + value: false, + trueAt: null, + })); + latestSnapshot = null; + version = 0; + }; + + return { + id: config.id, + label, + get version() { + return version; + }, + get results() { + return results; + }, + getLatestSnapshot: () => latestSnapshot, + clear: () => { + results = []; + latestSnapshot = null; + version = 0; + }, + observeFrame: (context) => { + ensureResults(context.runCount); + + let changed = latestSnapshot === null; + + context.forEachRunFrame((run) => { + const result = results[run.runIndex]; + if (!result || result.value || !shouldTestPredicateRun(run.status)) { + return; + } + + if ( + config.test({ + runIndex: run.runIndex, + status: run.status, + frame: run.frame, + }) + ) { + result.value = true; + result.trueAt = context.time; + changed = true; + } + }); + + if (!changed) { + return; + } + + latestSnapshot = createSnapshot({ + id: config.id, + label, + frameNumber: context.frameNumber, + time: context.time, + results, + }); + version++; + }, + }; +} diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/specs.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/specs.ts index 936d2f2b990..1f1eb5a1a91 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/specs.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/specs.ts @@ -2,11 +2,39 @@ import { createHirMetricEvaluator } from "../../frames/hir-metric"; import type { SDCPN } from "../../../types/sdcpn"; import type { + MonteCarloExpressionMetricSpec, MonteCarloMetricSpec, MonteCarloMetricSpecBase, + MonteCarloNumericMetricSpec, + MonteCarloPredicateSpec, MonteCarloUserDefinedMetricConfig, } from "./types"; +/** + * Splits an experiment's combined `metricSpecs` list into numeric metric + * specs and predicate specs. The two kinds share one channel (config field, + * init message); runtimes split once at the point they build executables. + */ +export function splitMonteCarloMetricSpecs( + specs: readonly MonteCarloMetricSpec[] | undefined, +): { + metricSpecs: MonteCarloNumericMetricSpec[]; + predicateSpecs: MonteCarloPredicateSpec[]; +} { + const metricSpecs: MonteCarloNumericMetricSpec[] = []; + const predicateSpecs: MonteCarloPredicateSpec[] = []; + + for (const spec of specs ?? []) { + if (spec.type === "Predicate") { + predicateSpecs.push(spec); + } else { + metricSpecs.push(spec); + } + } + + return { metricSpecs, predicateSpecs }; +} + function applyMetricSpecBase( spec: MonteCarloMetricSpecBase, measure: MonteCarloUserDefinedMetricConfig["measure"], @@ -27,7 +55,7 @@ function applyMetricSpecBase( } function createExpressionMetricConfig( - spec: Extract, + spec: MonteCarloExpressionMetricSpec, sdcpn: SDCPN, ): MonteCarloUserDefinedMetricConfig { // Expression metrics run exclusively as HIR-compiled buffer programs. @@ -48,7 +76,7 @@ function createExpressionMetricConfig( } export function createMonteCarloUserDefinedMetricConfigsFromSpecs( - specs: readonly MonteCarloMetricSpec[], + specs: readonly MonteCarloNumericMetricSpec[], sdcpn: SDCPN, ): MonteCarloUserDefinedMetricConfig[] { return specs.flatMap((spec) => { diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts index b89bf67c520..c1158c80fcc 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts @@ -71,6 +71,8 @@ export type MonteCarloUserDefinedMetricSampleRuns = export type MonteCarloMetricDistributionBinning = "exact" | { width: number }; +export type MonteCarloMetricType = "Metric" | "Predicate"; + export type MonteCarloMetricRunOutput = | { type: "scalar"; @@ -127,6 +129,11 @@ export type MonteCarloUserDefinedMetricConfig = { export type MonteCarloMetricSpecBase = { id: string; label: string; + /** + * Discriminates numeric metric specs from predicate specs when both travel + * in one experiment `metricSpecs` list. Absent means `"Metric"`. + */ + type?: "Metric"; sampleRuns?: MonteCarloUserDefinedMetricSampleRuns; runOutput?: MonteCarloMetricRunOutput; aggregateRuns?: MonteCarloUserDefinedMetricAggregation; @@ -161,11 +168,47 @@ export type MonteCarloExpressionMetricSpec = MonteCarloMetricSpecBase & { artifact: HirMetricArtifact; }; -export type MonteCarloMetricSpec = +/** Numeric (non-predicate) experiment metric specs. */ +export type MonteCarloNumericMetricSpec = | MonteCarloExpressionMetricSpec | MonteCarloPlaceTokenCountMeanMetricSpec | MonteCarloTransitionFiringCountMetricSpec; +/** + * A boolean condition evaluated per run: each run is tested on every frame + * until the predicate first returns true, then the time is recorded and the + * run is not tested again. Predicates deliberately do not carry the numeric + * sampling/aggregation options of metric specs. + */ +export type MonteCarloPredicateSpec = { + id: string; + label: string; + type: "Predicate"; + /** Predicates are always custom expression code. */ + kind: "expression"; + /** + * Function body over the same `state` object as expression metrics. It must + * `return` a boolean. Kept for display/persistence; execution uses only + * `artifact`. + */ + code: string; + /** + * HIR buffer program for `code`, compiled with a boolean return type + * (`compileHirArtifacts` + `metricReturnTypes`). This is the only execution + * path — specs without an artifact cannot run. + */ + artifact: HirMetricArtifact; +}; + +/** + * Everything an experiment can evaluate per frame. Numeric metrics and + * boolean predicates travel through the same `metricSpecs` channel; runtimes + * split them with `splitMonteCarloMetricSpecs` when building executables. + */ +export type MonteCarloMetricSpec = + | MonteCarloNumericMetricSpec + | MonteCarloPredicateSpec; + type MonteCarloUserDefinedMetricFrameBase = { metricId: string; label: string; @@ -221,3 +264,47 @@ export type MonteCarloUserDefinedMetric = MonteCarloFrameMetric & { getLatestFrame: () => MonteCarloUserDefinedMetricFrame | null; clear: () => void; }; + +export type MonteCarloUserDefinedPredicateMeasureInput = { + runIndex: number; + status: MonteCarloMetricRunStatus; + frame: SimulationFrameReader; +}; + +export type MonteCarloUserDefinedPredicateRunResult = { + runIndex: number; + value: boolean; + trueAt: number | null; +}; + +export type MonteCarloUserDefinedPredicateSnapshot = { + predicateId: string; + label: string; + frameNumber: number; + time: number; + /** Total runs, including errored runs (which can never become true). */ + runCount: number; + trueRunCount: number; + runResults: readonly MonteCarloUserDefinedPredicateRunResult[]; +}; + +export type MonteCarloUserDefinedPredicateConfig = { + id: string; + label?: string; + /** + * Tests whether one run satisfies the predicate at the current frame. + * + * A run is tested while the predicate is false for that run. Once it returns + * true, the true timestamp is recorded and the run is not tested again. + */ + test: (input: MonteCarloUserDefinedPredicateMeasureInput) => boolean; +}; + +export type MonteCarloUserDefinedPredicate = MonteCarloFrameMetric & { + readonly id: string; + readonly label: string; + readonly version: number; + readonly results: readonly MonteCarloUserDefinedPredicateRunResult[]; + getLatestSnapshot: () => MonteCarloUserDefinedPredicateSnapshot | null; + clear: () => void; +}; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.test.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.test.ts index 525c81e63ba..26f5fe9be05 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.test.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.test.ts @@ -6,7 +6,10 @@ import { createMonteCarloExperiment } from "./experiment"; import type { HirMetricArtifact } from "../../../hir/instantiate"; import type { SDCPN } from "../../../types/sdcpn"; import type { SimulationTransport } from "../../api"; -import type { MonteCarloUserDefinedMetricFrame } from "../metrics"; +import type { + MonteCarloUserDefinedMetricFrame, + MonteCarloUserDefinedPredicateSnapshot, +} from "../metrics"; import type { MonteCarloToMainMessage, MonteCarloToWorkerMessage, @@ -26,11 +29,16 @@ const empty = (): SDCPN => ({ function metricArtifact( code: string, sdcpn: SDCPN = empty(), + returnType: "number" | "boolean" = "number", ): HirMetricArtifact { - const { artifacts, failures } = compileHirArtifacts({ - ...sdcpn, - metrics: [{ id: "__test__", name: "test", code }], - }); + const { artifacts, failures } = compileHirArtifacts( + { + ...sdcpn, + metrics: [{ id: "__test__", name: "test", code }], + }, + undefined, + { metricReturnTypes: { __test__: returnType } }, + ); const artifact = artifacts.metrics.__test__; if (!artifact) { throw new Error( @@ -73,6 +81,27 @@ function makeMetricFrame( }; } +function makePredicateSnapshot( + trueRunCount: number, +): MonteCarloUserDefinedPredicateSnapshot { + return { + predicateId: "done", + label: "Done", + frameNumber: 1, + time: 1, + runCount: 2, + trueRunCount, + runResults: [ + { runIndex: 0, value: true, trueAt: 0 }, + { + runIndex: 1, + value: trueRunCount === 2, + trueAt: trueRunCount === 2 ? 1 : null, + }, + ], + }; +} + function makeMockTransport() { const sent: MonteCarloToWorkerMessage[] = []; const listeners = new Set<(message: unknown) => void>(); @@ -224,6 +253,53 @@ describe("createMonteCarloExperiment", () => { experiment.dispose(); }); + it("sends predicate metric specs to the worker and stores predicate snapshots", async () => { + const mock = makeMockTransport(); + const metricSpecs = [ + { + id: "done", + label: "Done", + type: "Predicate", + kind: "expression", + code: "return true;", + artifact: metricArtifact("return true;", empty(), "boolean"), + }, + ] as const; + const promise = createMonteCarloExperiment({ + transport: mock.transport, + sdcpn: empty(), + initialMarking: {}, + parameterValues: {}, + seed: 1, + dt: 1, + maxTime: 10, + runCount: 2, + metricSpecs, + }); + + // Predicate specs travel in-band in `metricSpecs`; the worker splits them. + expect(mock.sent[0]).toMatchObject({ + type: "init", + metricSpecs, + }); + + mock.simulate({ type: "ready" }); + const experiment = await promise; + const snapshot = makePredicateSnapshot(2); + mock.simulate({ + type: "predicateSnapshots", + snapshots: [snapshot], + }); + + expect(experiment.predicates.get()).toEqual({ + latestByPredicateId: { + done: snapshot, + }, + }); + + experiment.dispose(); + }); + it("runs locally when user-defined metrics are provided", async () => { const experiment = await createMonteCarloExperiment({ sdcpn: empty(), @@ -394,6 +470,121 @@ describe("createMonteCarloExperiment", () => { ); }); + it("checks local predicates until each run first becomes true", async () => { + const callsByRunIndex = new Map(); + const experiment = await createMonteCarloExperiment({ + sdcpn: empty(), + initialMarking: {}, + parameterValues: {}, + seed: 1, + dt: 1, + maxTime: 2, + runCount: 2, + metrics: [], + predicates: [ + { + id: "done", + label: "Done", + test: ({ frame, runIndex }) => { + callsByRunIndex.set(runIndex, [ + ...(callsByRunIndex.get(runIndex) ?? []), + frame.number, + ]); + + return frame.number >= runIndex; + }, + }, + ], + }); + + expect(experiment.predicates.get().latestByPredicateId.done).toEqual( + expect.objectContaining({ + trueRunCount: 1, + runResults: [ + { runIndex: 0, value: true, trueAt: 0 }, + { runIndex: 1, value: false, trueAt: null }, + ], + }), + ); + + const complete = new Promise((resolve) => { + experiment.events.subscribe((event) => { + if (event.type === "complete") { + resolve(); + } + }); + }); + + experiment.start(); + await complete; + + expect(experiment.predicates.get().latestByPredicateId.done).toEqual( + expect.objectContaining({ + trueRunCount: 2, + runResults: [ + { runIndex: 0, value: true, trueAt: 0 }, + { runIndex: 1, value: true, trueAt: 1 }, + ], + }), + ); + expect(callsByRunIndex.get(0)).toEqual([0]); + expect(callsByRunIndex.get(1)).toEqual([0, 1]); + }); + + it("evaluates compiled predicate artifacts against real frames", async () => { + const sdcpn: SDCPN = { + ...empty(), + places: [ + { + id: "place-done", + name: "Done", + colorId: null, + dynamicsEnabled: false, + differentialEquationId: null, + x: 0, + y: 0, + }, + ], + }; + const predicateSpec = (id: string, code: string) => + ({ + id, + label: id, + type: "Predicate", + kind: "expression", + code, + artifact: metricArtifact(code, sdcpn, "boolean"), + }) as const; + + const experiment = await createMonteCarloExperiment({ + sdcpn, + initialMarking: { "place-done": 2 }, + parameterValues: {}, + seed: 1, + dt: 1, + maxTime: 1, + runCount: 2, + metricSpecs: [ + predicateSpec("has-tokens", "return state.places.Done.count > 0;"), + predicateSpec("many-tokens", "return state.places.Done.count > 100;"), + ], + }); + + const { latestByPredicateId } = experiment.predicates.get(); + expect(latestByPredicateId["has-tokens"]).toEqual( + expect.objectContaining({ + trueRunCount: 2, + runResults: [ + { runIndex: 0, value: true, trueAt: 0 }, + { runIndex: 1, value: true, trueAt: 0 }, + ], + }), + ); + expect(latestByPredicateId["many-tokens"]).toEqual( + expect.objectContaining({ runCount: 2, trueRunCount: 0 }), + ); + }); + it("forwards start and cancel messages over the transport", async () => { const mock = makeMockTransport(); const promise = createExperimentWithMockTransport(mock); diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.ts index daed29c3f20..82630c30fb3 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/runtime/experiment.ts @@ -2,6 +2,9 @@ import { createWorkerTransport } from "../../runtime/transport"; import { createMonteCarloUserDefinedMetricConfigsFromSpecs, createMonteCarloUserDefinedMetric, + createMonteCarloUserDefinedPredicate, + createMonteCarloUserDefinedPredicateConfigsFromSpecs, + splitMonteCarloMetricSpecs, } from "../metrics"; import { createMonteCarloSimulator } from "../monte-carlo-simulator"; @@ -21,6 +24,9 @@ import type { MonteCarloUserDefinedMetric, MonteCarloUserDefinedMetricConfig, MonteCarloUserDefinedMetricFrame, + MonteCarloUserDefinedPredicate, + MonteCarloUserDefinedPredicateConfig, + MonteCarloUserDefinedPredicateSnapshot, } from "../metrics"; import type { MonteCarloAdvanceResult, MonteCarloSimulator } from "../types"; import type { @@ -41,6 +47,12 @@ export type MonteCarloExperimentMetrics = { latestByMetricId: Readonly>; }; +export type MonteCarloExperimentPredicates = { + latestByPredicateId: Readonly< + Record + >; +}; + export type MonteCarloExperimentEvent = | { type: "complete"; progress: MonteCarloWorkerProgress } | { type: "cancelled"; progress: MonteCarloWorkerProgress | null } @@ -69,16 +81,19 @@ export type CreateMonteCarloExperimentConfig = createWorker: WorkerFactory; transport?: never; metrics?: never; + predicates?: never; metricSpecs?: readonly MonteCarloMetricSpec[]; } | { transport: SimulationTransport; createWorker?: never; metrics?: never; + predicates?: never; metricSpecs?: readonly MonteCarloMetricSpec[]; } | { metrics: readonly MonteCarloUserDefinedMetricConfig[]; + predicates?: readonly MonteCarloUserDefinedPredicateConfig[]; createWorker?: never; transport?: never; metricSpecs?: never; @@ -88,6 +103,7 @@ export type CreateMonteCarloExperimentConfig = createWorker?: never; transport?: never; metrics?: never; + predicates?: never; } ); @@ -95,6 +111,7 @@ export interface MonteCarloExperiment { readonly status: ReadableStore; readonly progress: ReadableStore; readonly metrics: ReadableStore; + readonly predicates: ReadableStore; readonly events: EventStream; start(this: void): void; @@ -161,6 +178,12 @@ function createEmptyMetricsState(): MonteCarloExperimentMetrics { }; } +function createEmptyPredicatesState(): MonteCarloExperimentPredicates { + return { + latestByPredicateId: {}, + }; +} + function appendMetricFrames( state: MonteCarloExperimentMetrics, nextFrames: readonly MonteCarloUserDefinedMetricFrame[], @@ -177,6 +200,21 @@ function appendMetricFrames( }; } +function appendPredicateSnapshots( + state: MonteCarloExperimentPredicates, + snapshots: readonly MonteCarloUserDefinedPredicateSnapshot[], +): MonteCarloExperimentPredicates { + const latestByPredicateId = { ...state.latestByPredicateId }; + + for (const snapshot of snapshots) { + latestByPredicateId[snapshot.predicateId] = snapshot; + } + + return { + latestByPredicateId, + }; +} + function takePendingMetricFrames( metrics: readonly MonteCarloUserDefinedMetric[], lastFrameCounts: Map, @@ -189,6 +227,23 @@ function takePendingMetricFrames( }); } +function takePendingPredicateSnapshots( + predicates: readonly MonteCarloUserDefinedPredicate[], + lastVersions: Map, +): MonteCarloUserDefinedPredicateSnapshot[] { + return predicates.flatMap((predicate) => { + const lastVersion = lastVersions.get(predicate.id) ?? 0; + if (predicate.version === lastVersion) { + return []; + } + + lastVersions.set(predicate.id, predicate.version); + const snapshot = predicate.getLatestSnapshot(); + + return snapshot ? [snapshot] : []; + }); +} + function getLatestMetricFrame( metrics: readonly MonteCarloUserDefinedMetric[], ): MonteCarloUserDefinedMetricFrame | null { @@ -286,6 +341,7 @@ function getInitialProgress( function createLocalMonteCarloExperiment( config: CreateMonteCarloExperimentBaseConfig & { metrics: readonly MonteCarloUserDefinedMetricConfig[]; + predicates?: readonly MonteCarloUserDefinedPredicateConfig[]; }, ): Promise { const status = createReadableStore("Initializing"); @@ -293,6 +349,9 @@ function createLocalMonteCarloExperiment( const metrics = createReadableStore( createEmptyMetricsState(), ); + const predicates = createReadableStore( + createEmptyPredicatesState(), + ); const events = createEventStream(); let disposed = false; let running = false; @@ -302,7 +361,11 @@ function createLocalMonteCarloExperiment( const userMetrics = config.metrics.map((metricConfig) => createMonteCarloUserDefinedMetric(metricConfig), ); + const userPredicates = (config.predicates ?? []).map((predicateConfig) => + createMonteCarloUserDefinedPredicate(predicateConfig), + ); const lastMetricFrameCounts = new Map(); + const lastPredicateVersions = new Map(); const simulator = createMonteCarloSimulator({ sdcpn: config.sdcpn, extensions: config.extensions, @@ -313,7 +376,7 @@ function createLocalMonteCarloExperiment( maxTime: config.maxTime, hirArtifacts: config.hirArtifacts, runCount: config.runCount, - metrics: userMetrics, + metrics: [...userMetrics, ...userPredicates], }); const syncStores = (nextProgress: MonteCarloWorkerProgress | null) => { @@ -324,6 +387,15 @@ function createLocalMonteCarloExperiment( if (nextMetricFrames.length > 0) { metrics.set(appendMetricFrames(metrics.get(), nextMetricFrames)); } + const nextPredicateSnapshots = takePendingPredicateSnapshots( + userPredicates, + lastPredicateVersions, + ); + if (nextPredicateSnapshots.length > 0) { + predicates.set( + appendPredicateSnapshots(predicates.get(), nextPredicateSnapshots), + ); + } if (nextProgress) { progress.set(nextProgress); } @@ -392,6 +464,7 @@ function createLocalMonteCarloExperiment( status, progress, metrics, + predicates, events, start() { if (disposed || running) { @@ -468,11 +541,16 @@ export function createMonteCarloExperiment( !("transport" in config) ) { const { metricSpecs, ...baseConfig } = config; + const specs = splitMonteCarloMetricSpecs(metricSpecs); return createLocalMonteCarloExperiment({ ...baseConfig, metrics: createMonteCarloUserDefinedMetricConfigsFromSpecs( - metricSpecs, + specs.metricSpecs, + config.sdcpn, + ), + predicates: createMonteCarloUserDefinedPredicateConfigsFromSpecs( + specs.predicateSpecs, config.sdcpn, ), }); @@ -495,6 +573,9 @@ export function createMonteCarloExperiment( const metrics = createReadableStore( createEmptyMetricsState(), ); + const predicates = createReadableStore( + createEmptyPredicatesState(), + ); const events = createEventStream(); let disposed = false; @@ -553,6 +634,7 @@ export function createMonteCarloExperiment( status, progress, metrics, + predicates, events, start() { if (disposed) { @@ -588,6 +670,12 @@ export function createMonteCarloExperiment( metrics.set(appendMetricFrames(metrics.get(), message.frames)); break; } + case "predicateSnapshots": { + predicates.set( + appendPredicateSnapshots(predicates.get(), message.snapshots), + ); + break; + } case "progress": progress.set(message.progress); break; @@ -636,6 +724,7 @@ export function createMonteCarloExperiment( hirArtifacts: config.hirArtifacts, runCount: config.runCount, batchSize: config.batchSize, + // Predicate specs travel in-band; the worker splits them by `type`. metricSpecs: "metricSpecs" in config ? config.metricSpecs : undefined, }); } catch (error) { diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/messages.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/messages.ts index 4af3a65a41b..38feda9697b 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/messages.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/messages.ts @@ -5,6 +5,7 @@ import type { InitialMarking } from "../../api"; import type { MonteCarloMetricSpec, MonteCarloUserDefinedMetricFrame, + MonteCarloUserDefinedPredicateSnapshot, } from "../metrics"; import type { MonteCarloAdvanceResult } from "../types"; @@ -22,6 +23,7 @@ export type MonteCarloInitMessage = { hirArtifacts?: HirArtifacts; runCount: number; batchSize?: number; + /** Numeric metric and predicate specs; the worker splits them by `type`. */ metricSpecs?: readonly MonteCarloMetricSpec[]; }; @@ -48,6 +50,11 @@ export type MonteCarloMetricFramesMessage = { frames: MonteCarloUserDefinedMetricFrame[]; }; +export type MonteCarloPredicateSnapshotsMessage = { + type: "predicateSnapshots"; + snapshots: MonteCarloUserDefinedPredicateSnapshot[]; +}; + export type MonteCarloReadyMessage = { type: "ready"; }; @@ -78,6 +85,7 @@ export type MonteCarloToMainMessage = | MonteCarloReadyMessage | MonteCarloProgressMessage | MonteCarloMetricFramesMessage + | MonteCarloPredicateSnapshotsMessage | MonteCarloCompleteMessage | MonteCarloCancelledMessage | MonteCarloErrorMessage; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/monte-carlo.worker.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/monte-carlo.worker.ts index 4d3e29827dc..bdf1622753a 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/monte-carlo.worker.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/worker/monte-carlo.worker.ts @@ -3,12 +3,16 @@ import { SDCPNItemError } from "../../../errors"; import { createMonteCarloUserDefinedMetric, createMonteCarloUserDefinedMetricConfigsFromSpecs, + createMonteCarloUserDefinedPredicate, + createMonteCarloUserDefinedPredicateConfigsFromSpecs, + splitMonteCarloMetricSpecs, } from "../metrics"; import { createMonteCarloSimulator } from "../monte-carlo-simulator"; import type { MonteCarloUserDefinedMetric, MonteCarloUserDefinedMetricFrame, + MonteCarloUserDefinedPredicate, } from "../metrics"; import type { MonteCarloAdvanceResult, MonteCarloSimulator } from "../types"; import type { @@ -27,10 +31,12 @@ const DEFAULT_BATCH_SIZE = 4; let simulator: MonteCarloSimulator | null = null; let userMetrics: MonteCarloUserDefinedMetric[] = []; +let userPredicates: MonteCarloUserDefinedPredicate[] = []; let isRunning = false; let isInitialized = false; let batchSize = DEFAULT_BATCH_SIZE; let lastSentMetricFrameCounts = new Map(); +let lastSentPredicateVersions = new Map(); let latestProgress: MonteCarloWorkerProgress | null = null; function postTypedMessage(message: MonteCarloToMainMessage): void { @@ -141,14 +147,40 @@ function postPendingMetricFrames(): void { } } +function postPendingPredicateSnapshots(): void { + if (userPredicates.length === 0) { + return; + } + + const snapshots = userPredicates.flatMap((predicate) => { + const lastSentVersion = lastSentPredicateVersions.get(predicate.id) ?? 0; + if (predicate.version === lastSentVersion) { + return []; + } + + lastSentPredicateVersions.set(predicate.id, predicate.version); + const snapshot = predicate.getLatestSnapshot(); + + return snapshot ? [snapshot] : []; + }); + + if (snapshots.length > 0) { + postTypedMessage({ type: "predicateSnapshots", snapshots }); + } +} + function initialize(message: MonteCarloInitMessage): void { - const metricSpecs = message.metricSpecs; - userMetrics = metricSpecs - ? createMonteCarloUserDefinedMetricConfigsFromSpecs( - metricSpecs, - message.sdcpn, - ).map((metricConfig) => createMonteCarloUserDefinedMetric(metricConfig)) - : []; + const specs = splitMonteCarloMetricSpecs(message.metricSpecs); + userMetrics = createMonteCarloUserDefinedMetricConfigsFromSpecs( + specs.metricSpecs, + message.sdcpn, + ).map((metricConfig) => createMonteCarloUserDefinedMetric(metricConfig)); + userPredicates = createMonteCarloUserDefinedPredicateConfigsFromSpecs( + specs.predicateSpecs, + message.sdcpn, + ).map((predicateConfig) => + createMonteCarloUserDefinedPredicate(predicateConfig), + ); simulator = createMonteCarloSimulator({ sdcpn: message.sdcpn, extensions: message.extensions, @@ -159,16 +191,18 @@ function initialize(message: MonteCarloInitMessage): void { maxTime: message.maxTime, hirArtifacts: message.hirArtifacts, runCount: message.runCount, - metrics: userMetrics, + metrics: [...userMetrics, ...userPredicates], }); batchSize = message.batchSize ?? DEFAULT_BATCH_SIZE; isInitialized = true; isRunning = false; lastSentMetricFrameCounts = new Map(); + lastSentPredicateVersions = new Map(); latestProgress = initialProgress(message.runCount); postTypedMessage({ type: "ready" }); postPendingMetricFrames(); + postPendingPredicateSnapshots(); postTypedMessage({ type: "progress", progress: latestProgress }); } @@ -191,6 +225,7 @@ async function computeLoop(): Promise { if (result) { latestProgress = progressFromResult(result); postPendingMetricFrames(); + postPendingPredicateSnapshots(); postTypedMessage({ type: "progress", progress: latestProgress }); if (result.allFinished) { @@ -214,6 +249,7 @@ workerRuntime.onMessage((message) => { isRunning = false; simulator = null; userMetrics = []; + userPredicates = []; postTypedMessage({ type: "error", message: diff --git a/libs/@hashintel/petrinaut-core/src/workers/monte-carlo.ts b/libs/@hashintel/petrinaut-core/src/workers/monte-carlo.ts index e252c95ebb1..c0cee435d8e 100644 --- a/libs/@hashintel/petrinaut-core/src/workers/monte-carlo.ts +++ b/libs/@hashintel/petrinaut-core/src/workers/monte-carlo.ts @@ -6,6 +6,7 @@ export type { MonteCarloErrorMessage, MonteCarloInitMessage, MonteCarloMetricFramesMessage, + MonteCarloPredicateSnapshotsMessage, MonteCarloProgressMessage, MonteCarloReadyMessage, MonteCarloStartMessage, diff --git a/libs/@hashintel/petrinaut/docs/experiments.md b/libs/@hashintel/petrinaut/docs/experiments.md index 3012b1eacb0..e39484706dc 100644 --- a/libs/@hashintel/petrinaut/docs/experiments.md +++ b/libs/@hashintel/petrinaut/docs/experiments.md @@ -26,6 +26,18 @@ The model used is a snapshot of the current net at the time you press **Run**. E > Currently, an experiment can only run against one scenario at a time. To compare scenarios, create one experiment per scenario. +### Metrics + +Each experiment must include at least one metric or predicate. + +Metrics return values from the current frame. Add a built-in metric such as a place token count or transition firing count, choose an existing model metric, or add custom metric code. + +Custom code metrics can also be marked as a **Predicate**. Turn on the **Predicate** toggle in a custom code metric when the code should return `true` or `false` instead of a number. Predicate code uses the same `state.places["Place Name"].count` and `state.places["Place Name"].tokens` object as numeric metric code, but it must return a boolean. + +Each run starts with the predicate set to false. Petrinaut checks it on every frame until it first returns true for that run, records the time it became true, and then stops checking that predicate for that run. Runs that end in an error are never marked true. + +In the experiment drawer, numeric metrics appear as timeline boxes that can be expanded. Predicates appear in the same Metrics section as compact boxes showing the percentage of runs that have reached the predicate, the run count behind it (for example "7 of 10 runs"), and — once at least one run is true — the median simulation time at which runs first satisfied it. The percentage is rounded down, so 100% always means every run reached the predicate. + ## Lifecycle and statuses Experiments progress through five status labels: diff --git a/libs/@hashintel/petrinaut/src/react/experiments/context.ts b/libs/@hashintel/petrinaut/src/react/experiments/context.ts index 46fea6e6099..24ebb3f75e4 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/context.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/context.ts @@ -2,8 +2,11 @@ import { createContext } from "react"; import type { MonteCarloExpressionMetricSpec, - MonteCarloMetricSpec, + MonteCarloPlaceTokenCountMeanMetricSpec, + MonteCarloPredicateSpec, + MonteCarloTransitionFiringCountMetricSpec, MonteCarloUserDefinedMetricFrame, + MonteCarloUserDefinedPredicateSnapshot, MonteCarloWorkerProgress, } from "@hashintel/petrinaut-core"; @@ -15,13 +18,16 @@ export type ExperimentStatus = | "cancelled"; /** - * Metric spec as authored by the experiment form. Expression metrics are - * provided without a compiled `artifact` — the experiments provider compiles - * them through the HIR (in the language worker) before starting the run. + * Metric or predicate spec as authored by the experiment form. Expression + * metrics and predicates are provided without a compiled `artifact` — the + * experiments provider compiles them through the HIR (in the language worker) + * before starting the run. */ export type ExperimentMetricSpecInput = - | Exclude - | Omit; + | MonteCarloPlaceTokenCountMeanMetricSpec + | MonteCarloTransitionFiringCountMetricSpec + | Omit + | Omit; export type CreateExperimentInput = { name: string; @@ -52,6 +58,9 @@ export type ExperimentRecord = { latestMetricFramesById: Readonly< Record >; + latestPredicateSnapshotsById: Readonly< + Record + >; }; export function isExperimentActive(experiment: ExperimentRecord): boolean { diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx index 112573ab763..dfca1001254 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { type MonteCarloUserDefinedMetricFrame, + type MonteCarloUserDefinedPredicateSnapshot, DEFAULT_PETRINAUT_EXTENSIONS, type SDCPN, type WorkerLike, @@ -79,6 +80,21 @@ function makeMetricFrame(): MonteCarloUserDefinedMetricFrame { }; } +function makePredicateSnapshot(): MonteCarloUserDefinedPredicateSnapshot { + return { + predicateId: "done", + label: "Done", + frameNumber: 0, + time: 0, + runCount: 2, + trueRunCount: 1, + runResults: [ + { runIndex: 0, value: true, trueAt: 0 }, + { runIndex: 1, value: false, trueAt: null }, + ], + }; +} + class FakeMonteCarloWorker { readonly sent: MonteCarloToWorkerMessage[] = []; readonly postMessage = vi.fn((message: MonteCarloToWorkerMessage) => { @@ -155,8 +171,8 @@ const LanguageClientOverride = ({ children }: React.PropsWithChildren) => { - Promise.resolve(compileHirArtifacts(sdcpn, extensions)), + requestHirArtifacts: (sdcpn, extensions, options) => + Promise.resolve(compileHirArtifacts(sdcpn, extensions, options)), }} > {children} @@ -586,6 +602,73 @@ describe("ExperimentsProvider", () => { } }); + it("runs experiment predicate metrics in the worker", async () => { + const worker = new FakeMonteCarloWorker(); + const { getValue, renderResult } = renderExperimentsProvider(worker); + const metricSpecs = [ + { + id: "done", + label: "Done", + type: "Predicate", + kind: "expression", + code: "return true;", + }, + ] as const; + + try { + await act(async () => { + const createPromise = getValue().createExperiment({ + name: "Predicate experiment", + scenarioId: null, + scenarioParameterValues: {}, + runCount: 2, + seed: 42, + dt: 1, + maxTime: 1, + metricSpecs, + }); + + await flushWorkerSetup(); + const initMessage = worker.sent[0]; + // The provider compiles the predicate and sends it in-band with a + // boolean artifact attached; the worker splits specs by `type`. + expect(worker.sent[0]).toMatchObject({ + type: "init", + metricSpecs: [ + { + id: "done", + label: "Done", + type: "Predicate", + kind: "expression", + code: "return true;", + }, + ], + }); + if (initMessage?.type !== "init") { + throw new Error("Expected init message"); + } + const sentSpec = initMessage.metricSpecs?.[0]; + if (sentSpec?.type !== "Predicate") { + throw new Error("Expected a predicate spec"); + } + expect(typeof sentSpec.artifact.source).toBe("string"); + worker.emit({ type: "ready" }); + await createPromise; + }); + + const snapshot = makePredicateSnapshot(); + await act(async () => { + worker.emit({ type: "predicateSnapshots", snapshots: [snapshot] }); + }); + + expect( + getValue().selectedExperiment?.latestPredicateSnapshotsById.done, + ).toEqual(snapshot); + } finally { + renderResult.unmount(); + } + }); + it("notifies when a Monte Carlo experiment errors", async () => { const addNotification = vi.fn(() => "notification-id"); const worker = new FakeMonteCarloWorker(); diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx index e93d0e78ef4..04eff9953df 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx @@ -7,6 +7,7 @@ import { type InitialMarking, type MonteCarloExperiment, type MonteCarloExperimentState, + type MonteCarloMetricSpec, type WorkerFactory, type Scenario, type ScenarioParameter, @@ -142,19 +143,20 @@ function assertExperimentInput(input: CreateExperimentInput): void { const metricIds = new Set(); for (const metricSpec of input.metricSpecs) { + const itemType = metricSpec.type === "Predicate" ? "Predicate" : "Metric"; const metricId = metricSpec.id.trim(); if (metricId === "") { - throw new Error("Metric id is required"); + throw new Error(`${itemType} id is required`); } if (metricIds.has(metricId)) { - throw new Error(`Metric id "${metricId}" is duplicated`); + throw new Error(`${itemType} id "${metricId}" is duplicated`); } metricIds.add(metricId); if (metricSpec.label.trim() === "") { - throw new Error("Metric label is required"); + throw new Error(`${itemType} label is required`); } if (metricSpec.kind === "expression" && metricSpec.code.trim() === "") { - throw new Error(`Metric "${metricSpec.label}" code is required`); + throw new Error(`${itemType} "${metricSpec.label}" code is required`); } } } @@ -237,6 +239,8 @@ export const ExperimentsProvider: React.FC = ({ const sync = () => { patchExperiment(experimentId, { latestMetricFramesById: handle.metrics.get().latestByMetricId, + latestPredicateSnapshotsById: + handle.predicates.get().latestByPredicateId, metricFrames: handle.metrics.get().frames, progress: handle.progress.get(), status: mapExperimentStatus(handle.status.get()), @@ -246,6 +250,7 @@ export const ExperimentsProvider: React.FC = ({ const unsubscribeStatus = handle.status.subscribe(sync); const unsubscribeProgress = handle.progress.subscribe(sync); const unsubscribeMetrics = handle.metrics.subscribe(sync); + const unsubscribePredicates = handle.predicates.subscribe(sync); const unsubscribeEvents = handle.events.subscribe((event) => { if (event.type === "error") { patchExperiment(experimentId, { @@ -278,6 +283,7 @@ export const ExperimentsProvider: React.FC = ({ unsubscribeStatus(); unsubscribeProgress(); unsubscribeMetrics(); + unsubscribePredicates(); unsubscribeEvents(); }, }); @@ -352,6 +358,7 @@ export const ExperimentsProvider: React.FC = ({ metricSpecs: input.metricSpecs, progress: null, latestMetricFramesById: {}, + latestPredicateSnapshotsById: {}, metricFrames: [], }; @@ -367,43 +374,58 @@ export const ExperimentsProvider: React.FC = ({ // worker — the simulation engine has no compiler of its own. The // experiment's expression metrics are compiled alongside by // substituting them for the model's metrics. - const expressionSpecs = input.metricSpecs.filter( + const expressionMetricSpecs = input.metricSpecs.filter( (spec) => spec.kind === "expression", ); + const expressionPredicateSpecs = expressionMetricSpecs.filter( + (spec) => spec.type === "Predicate", + ); const { artifacts, failures } = await requestHirArtifacts( { ...experimentSdcpn, - metrics: expressionSpecs.map((spec) => ({ + metrics: expressionMetricSpecs.map((spec) => ({ id: spec.id, name: spec.label, code: spec.code, })), }, extensionsRef.current, + { + metricReturnTypes: Object.fromEntries( + expressionPredicateSpecs.map((spec) => [ + spec.id, + "boolean" as const, + ]), + ), + }, ); - const metricSpecs = input.metricSpecs.map((spec) => { - if (spec.kind !== "expression") { - return spec; - } - const artifact = artifacts.metrics[spec.id]; - if (!artifact) { - const diagnostics = failures - .filter( - (failure) => - failure.itemType === "metric" && failure.itemId === spec.id, - ) - .flatMap((failure) => - failure.diagnostics.map((diagnostic) => diagnostic.message), + const metricSpecs = input.metricSpecs.map( + (spec): MonteCarloMetricSpec => { + if (spec.kind !== "expression") { + return spec; + } + const artifact = artifacts.metrics[spec.id]; + if (!artifact) { + const diagnostics = failures + .filter( + (failure) => + failure.itemType === "metric" && failure.itemId === spec.id, + ) + .flatMap((failure) => + failure.diagnostics.map((diagnostic) => diagnostic.message), + ); + const itemType = + spec.type === "Predicate" ? "Predicate" : "Metric"; + throw new Error( + `${itemType} "${spec.label}" did not compile${ + diagnostics.length > 0 ? `: ${diagnostics.join("; ")}` : "" + }`, ); - throw new Error( - `Metric "${spec.label}" did not compile${ - diagnostics.length > 0 ? `: ${diagnostics.join("; ")}` : "" - }`, - ); - } - return { ...spec, artifact }; - }); + } + return { ...spec, artifact }; + }, + ); const experimentConfigBase = { sdcpn: experimentSdcpn, diff --git a/libs/@hashintel/petrinaut/src/react/lsp/context.ts b/libs/@hashintel/petrinaut/src/react/lsp/context.ts index ee3aefbf357..fe0fe0a5ba9 100644 --- a/libs/@hashintel/petrinaut/src/react/lsp/context.ts +++ b/libs/@hashintel/petrinaut/src/react/lsp/context.ts @@ -5,6 +5,7 @@ import type { Diagnostic, DocumentUri, HirCompileResult, + HirCompileOptions, Hover, PetrinautExtensionSettings, Position, @@ -49,6 +50,7 @@ export interface LanguageClientContextValue { requestHirArtifacts: ( sdcpn: SDCPN, extensions?: PetrinautExtensionSettings, + options?: HirCompileOptions, ) => Promise; /** Initialize a temporary scenario editing session. */ initializeScenarioSession: (params: ScenarioSessionParams) => void; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx index 4df9581df1e..b6a60578096 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx @@ -1,5 +1,12 @@ import { Collapsible } from "@ark-ui/react/collapsible"; -import { use, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { + use, + useEffect, + useLayoutEffect, + useRef, + useState, + type ReactNode, +} from "react"; import { Button, @@ -8,6 +15,7 @@ import { NumberInput, Select, TextInput, + Toggle, type SelectItem, } from "@hashintel/ds-components"; import { css, cx } from "@hashintel/ds-helpers/css"; @@ -197,6 +205,19 @@ const metricSpecificFieldsStyle = css({ gap: "2", }); +const codeHeaderStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + gap: "2", +}); + +const predicateToggleInlineStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + const codeDiagnosticStyle = css({ fontSize: "xs", color: "red.s100", @@ -243,6 +264,22 @@ const DEFAULT_METRIC_CODE = `/** */ return 0;`; +const DEFAULT_PREDICATE_CODE = `/** +* Predicate code that will be checked on each frame until it returns true for a run. +* It must \`return\` a boolean. +* +* The only thing in scope is \`state\`, a snapshot of the current frame: +* state.places["Place Name"].count -> number of tokens in a place +* state.places["Place Name"].tokens -> array of token objects (for colored places) +* +* Reference places by their exact name. Use bracket access for names +* with spaces, e.g. state.places["Work In Progress"]. +* +* --- Example: true once all work has completed --- +* return state.places["Work In Progress"].count === 0; +*/ + +return false;`; const EMPTY_SCENARIOS: readonly Scenario[] = []; function getDefaultScenarioSelection(scenarios: readonly Scenario[]): string { @@ -284,6 +321,7 @@ type TransitionFiringMode = NonNullable< type ExperimentMetricDraft = { id: string; + type: "Metric" | "Predicate"; kind: ExperimentMetricKind; label: string; expanded: boolean; @@ -319,6 +357,10 @@ function getMetricSummaryLabel( metric: ExperimentMetricDraft, sdcpn: SDCPN, ): string { + if (metric.type === "Predicate") { + return "Predicate"; + } + if (metric.sourceMetricId) { const modelMetric = sdcpn.metrics?.find( (candidate) => candidate.id === metric.sourceMetricId, @@ -356,6 +398,7 @@ function canReplaceMetricLabel(label: string, sdcpn: SDCPN): boolean { return new Set([ "", "Custom metric", + "Predicate", "Place tokens", "Transition firing", getDefaultMetricLabel("placeTokenCountMean", sdcpn), @@ -419,6 +462,21 @@ function getMetricKindIcon( return METRIC_KIND_ICONS[value]; } +function getCodeForMetricTypeChange( + code: string, + nextType: ExperimentMetricDraft["type"], +): string { + if (nextType === "Predicate" && code.trim() === DEFAULT_METRIC_CODE.trim()) { + return DEFAULT_PREDICATE_CODE; + } + + if (nextType === "Metric" && code.trim() === DEFAULT_PREDICATE_CODE.trim()) { + return DEFAULT_METRIC_CODE; + } + + return code; +} + function createDefaultMetricDraft(sdcpn: SDCPN): ExperimentMetricDraft { const place = sdcpn.places[0]; const transition = sdcpn.transitions[0]; @@ -430,6 +488,7 @@ function createDefaultMetricDraft(sdcpn: SDCPN): ExperimentMetricDraft { return { id: crypto.randomUUID(), + type: "Metric", kind, label: getDefaultMetricLabel(kind, sdcpn), expanded: true, @@ -448,7 +507,7 @@ function buildMetricSpecs( sdcpn: SDCPN, ): ExperimentMetricSpecInput[] { if (drafts.length === 0) { - throw new Error("Define at least one metric"); + return []; } return drafts.map((draft, index) => { @@ -458,9 +517,11 @@ function buildMetricSpecs( throw new Error(`Metric ${index + 1} needs a label`); } + const itemType = draft.type === "Predicate" ? "Predicate" : "Metric"; const sampledMetricBase = { id: draft.id, label, + type: "Metric" as const, sampleRuns: "all" as const, runOutput: { type: "distribution" as const }, }; @@ -495,12 +556,22 @@ function buildMetricSpecs( } case "expression": { if (draft.code.trim() === "") { - throw new Error(`Metric "${label}" code is required`); + throw new Error(`${itemType} "${label}" code is required`); } // Compilation happens through the HIR when the experiment starts // (the provider attaches the compiled artifact); live validation is // covered by the metric LSP session diagnostics. + if (draft.type === "Predicate") { + return { + id: draft.id, + label, + type: "Predicate", + kind: "expression", + code: draft.code, + }; + } + return { ...sampledMetricBase, kind: "expression", @@ -542,13 +613,15 @@ const ScenarioParameterRow = ({ const ExperimentMetricLspSession = ({ code, metricSessionId, + returnType, onChange, }: { code: string; metricSessionId: string; + returnType: "number" | "boolean"; onChange: (diagnostics: MetricLspDiagnosticSummary) => void; }) => { - useMetricLspSession(code, metricSessionId); + useMetricLspSession(code, metricSessionId, returnType); const { diagnosticsByUri } = use(LanguageClientContext); const { count, firstMessage } = summarizeMetricLspErrors( diagnosticsByUri, @@ -567,12 +640,14 @@ const ExperimentExpressionMetricEditor = ({ code, metricSessionId, lspDiagnostics, + headerEnd, readOnly = false, onChange, }: { code: string; metricSessionId: string; lspDiagnostics: MetricLspDiagnosticSummary; + headerEnd?: ReactNode; readOnly?: boolean; onChange: (code: string) => void; }) => { @@ -580,7 +655,10 @@ const ExperimentExpressionMetricEditor = ({ return (
- Code +
+ Code + {headerEnd} +
= { + type: nextKind === "expression" ? metric.type : "Metric", kind: nextKind, label: nextLabel, sourceMetricId: null, @@ -707,6 +787,7 @@ const ExperimentMetricRow = ({ ) : null} @@ -834,6 +915,34 @@ const ExperimentMetricRow = ({ code={metric.code} metricSessionId={metric.metricSessionId} lspDiagnostics={metric.lspDiagnostics} + headerEnd={ + metric.sourceMetricId === null ? ( +
+ Predicate + { + const nextType = checked ? "Predicate" : "Metric"; + updateMetric({ + type: nextType, + code: getCodeForMetricTypeChange( + metric.code, + nextType, + ), + label: canReplaceMetricLabel(metric.label, sdcpn) + ? nextType === "Predicate" + ? "Predicate" + : getDefaultMetricLabel("expression", sdcpn) + : metric.label, + lspDiagnostics: EMPTY_METRIC_LSP_DIAGNOSTICS, + }); + }} + /> +
+ ) : undefined + } readOnly={metric.sourceMetricId !== null} onChange={(code) => updateMetric({ code })} /> @@ -890,12 +999,12 @@ export const CreateExperimentDrawer = ({ const metricKindGroups = createMetricKindGroups(petriNetDefinition); const metricDiagnosticError = getExperimentMetricDiagnosticError(metricDrafts); - const metricFormError = + const itemFormError = metricDrafts.length === 0 ? "Define at least one metric" : metricDiagnosticError; - const footerError = error ?? metricFormError; - const canRun = !isSubmitting && metricFormError === null; + const footerError = error ?? itemFormError; + const canRun = !isSubmitting && itemFormError === null; const resetForm = () => { setName(DEFAULT_EXPERIMENT_NAME); @@ -984,7 +1093,7 @@ export const CreateExperimentDrawer = ({ setError(null); - if (metricDiagnosticError) { + if (itemFormError) { return; } @@ -1184,7 +1293,7 @@ export const CreateExperimentDrawer = ({ tone="neutral" size="sm" disabled={!canRun} - tooltip={metricFormError ?? undefined} + tooltip={itemFormError ?? undefined} prefix={ isSubmitting ? ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-lsp-validation.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-lsp-validation.ts index 83b68990fe2..4a79c4532dd 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-lsp-validation.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-lsp-validation.ts @@ -5,6 +5,7 @@ export type MetricLspDiagnosticSummary = { export type ExperimentMetricDiagnosticDraft = { kind: string; + type?: "Metric" | "Predicate"; label: string; lspDiagnostics: MetricLspDiagnosticSummary; }; @@ -34,9 +35,13 @@ export function getExperimentMetricDiagnosticError( const diagnosticCount = firstMetricWithDiagnostics.lspDiagnostics.count; const firstMessage = firstMetricWithDiagnostics.lspDiagnostics.firstMessage; - const label = firstMetricWithDiagnostics.label.trim() || "Untitled metric"; + const itemType = + firstMetricWithDiagnostics.type === "Predicate" ? "Predicate" : "Metric"; + const label = + firstMetricWithDiagnostics.label.trim() || + (itemType === "Predicate" ? "Untitled predicate" : "Untitled metric"); - return `Metric "${label}" has ${diagnosticCount} code diagnostic${ + return `${itemType} "${label}" has ${diagnosticCount} code diagnostic${ diagnosticCount === 1 ? "" : "s" }${firstMessage ? `: ${firstMessage}` : "."}`; } diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx index e0c2031b920..829279a0183 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx @@ -96,6 +96,7 @@ export function makeExperiment( frameNumber: status === "complete" ? 180 : 45, }), latestMetricFramesById: {}, + latestPredicateSnapshotsById: {}, metricFrames: [], ...overrides, }; @@ -163,6 +164,7 @@ const createFakeExperiment = ( metricSpecs: input.metricSpecs, progress: null, latestMetricFramesById: {}, + latestPredicateSnapshotsById: {}, metricFrames: [], }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.tsx index c9f046d1023..bc93ea0e5ee 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.tsx @@ -93,6 +93,34 @@ const metricItemLargeStyle = css({ gridColumn: "[1 / -1]", }); +const predicateHeaderStyle = css({ + display: "flex", + alignItems: "baseline", +}); + +const predicateTitleStyle = css({ + fontSize: "sm", + fontWeight: "semibold", + color: "neutral.s120", + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}); + +const predicatePercentageStyle = css({ + fontSize: "[36px]", + fontWeight: "semibold", + fontVariantNumeric: "tabular-nums", + color: "neutral.s120", + lineHeight: "[1]", +}); + +const predicateDetailStyle = css({ + fontSize: "xs", + color: "neutral.s80", + fontVariantNumeric: "tabular-nums", +}); + const footerSpacerStyle = css({ flex: "1", }); @@ -118,6 +146,30 @@ function formatStatus(experiment: ExperimentRecord): string { } } +function median(values: readonly number[]): number | null { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 + ? (sorted[middle - 1]! + sorted[middle]!) / 2 + : sorted[middle]!; +} + +/** Latest predicate snapshots, in the order the experiment defined them. */ +function getOrderedPredicateSnapshots(experiment: ExperimentRecord) { + return experiment.metricSpecs.flatMap((spec) => { + if (spec.type !== "Predicate") { + return []; + } + const snapshot = experiment.latestPredicateSnapshotsById[spec.id]; + return snapshot ? [snapshot] : []; + }); +} + function groupMetricFramesByMetric( metricFrames: readonly MetricFrame[], ): MetricFrame[][] { @@ -200,8 +252,9 @@ const ExperimentMetrics = ({ }) => { const [sizes, setSizes] = useState>({}); const metricFrameGroups = groupMetricFramesByMetric(experiment.metricFrames); + const predicateSnapshots = getOrderedPredicateSnapshots(experiment); - if (metricFrameGroups.length === 0) { + if (metricFrameGroups.length === 0 && predicateSnapshots.length === 0) { return null; } @@ -232,6 +285,33 @@ const ExperimentMetrics = ({
); })} + {predicateSnapshots.map((snapshot) => { + // Floored so 100% only ever means every single run reached true. + const percentage = + snapshot.runCount === 0 + ? 0 + : Math.floor((snapshot.trueRunCount / snapshot.runCount) * 100); + const medianTrueAt = median( + snapshot.runResults.flatMap((result) => + result.trueAt === null ? [] : [result.trueAt], + ), + ); + + return ( +
+
+ {snapshot.label} +
+ {percentage}% + + {snapshot.trueRunCount} of {snapshot.runCount} runs + {medianTrueAt !== null + ? ` · median t = ${formatNumber(medianTrueAt)}` + : ""} + +
+ ); + })} ); }; @@ -269,7 +349,8 @@ export const ViewExperimentDrawer = ({
- {experiment.metricFrames.length > 0 ? ( + {experiment.metricFrames.length > 0 || + Object.keys(experiment.latestPredicateSnapshotsById).length > 0 ? (
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/metrics/metric-form.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/metrics/metric-form.tsx index 7a013529985..32e5802bdd2 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/metrics/metric-form.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/metrics/metric-form.tsx @@ -121,6 +121,7 @@ export type MetricFormInstance = ReturnType; export function useMetricLspSession( code: string, providedSessionId?: string, + returnType: "number" | "boolean" = "number", ): string { const { initializeMetricSession, updateMetricSession, killMetricSession } = use(LanguageClientContext); @@ -130,7 +131,7 @@ export function useMetricLspSession( const initializedRef = useRef(false); useEffect(() => { - const sessionData = { sessionId, code }; + const sessionData = { sessionId, code, returnType }; if (!initializedRef.current) { initializeMetricSession(sessionData); @@ -138,7 +139,13 @@ export function useMetricLspSession( } else { updateMetricSession(sessionData); } - }, [code, initializeMetricSession, sessionId, updateMetricSession]); + }, [ + code, + initializeMetricSession, + returnType, + sessionId, + updateMetricSession, + ]); useEffect(() => { return () => {