From 5453ca20b3e5e4a8b513bfe22cdb8507d13104ac Mon Sep 17 00:00:00 2001 From: vinodhalaharvi-claude Date: Sun, 24 May 2026 13:22:05 +0000 Subject: [PATCH] scriptmem: attach the in-memory runtime to the unified pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of the unification: a memory-backend block now EXECUTES. Until now a 'memory' block resolved (vocabulary + availability) but dead-ended at Finalize, which only produces a temporal Sibyl Plan. This adds the memory interpreter branch — the existing in-process runtime, unchanged, reached through the same Parse → Resolve front as the temporal path. The two backends now share one front end and diverge only at the final interpreter: temporal → Lower → Finalize → sibyl.Plan → submit (durable) memory → adapt resolved.AST → *Program → runtime.Execute (in-process) Critically, this is purely ADDITIVE and the runtime is untouched: - internal/agentscript (Execute, all 44 verbs, plugins) is not modified. - The bridge ADAPTS the unified pipeline's resolved.AST into the old *Program the runtime already consumes, then calls the existing Execute. The proven interpreter runs exactly as it always has. New package: pkg/scriptmem (deliberately SEPARATE from pkg/script). - RunMemory(ctx, MemoryConfig, resolved.AST) (string, error) — the synchronous sibling of the temporal submit path: runs the resolved memory program in-process and returns the result directly. Rejects a non-memory program so a temporal block is never run in-process. - toProgram / nodeToStatement / pipelineToStatement / parallelToStatement / callToCommand — the adapter shim: Pipeline → *Statement chain linked by .Pipe (a >=> b >=> c) Parallel → *Statement{Parallel: branches} (a <*> b) Call → *Command{Action, Arg..Arg4} (≤4 positional args) WHY A SEPARATE PACKAGE (important): the in-memory runtime drags a heavy dependency tree (Google Workspace, GitHub, Gemini, oauth2 SDKs). If the bridge lived in pkg/script, every consumer of the compile pipeline — loom included — would transitively pull all of it just to translate or compile, even when they never run memory. Verified: with the bridge in pkg/script, loom failed to build (missing go.sum for oauth2 / google api). Moving it to pkg/scriptmem keeps pkg/script lean; loom builds clean against this branch with no new deps. You only pay for the runtime's deps if you actually execute memory. Tests (pkg/scriptmem/memory_adapter_test.go): run the real front end (script.Parse → script.Resolve over CompleteRegistry) then assert the adapter output — - single call maps name + first arg correctly; - a >=> b >=> c folds into the correct .Pipe-linked *Statement chain; - RunMemory rejects a temporal-backend program. (Structural translation is tested without a live runtime, which would need API keys; execution itself reuses the long-standing, separately exercised Execute.) loom unaffected and lean: pkg/script carries no runtime import; built + tested loom against this commit, clean. Next: loom's small sync branch — when it submits a memory-flagged query, post the RunMemory result immediately (vs the temporal correlate-and- await path it already has). CI: vet, gofmt, staticcheck, go test -race ./..., go build ./... pass. --- pkg/scriptmem/memory.go | 189 +++++++++++++++++++++++++++ pkg/scriptmem/memory_adapter_test.go | 75 +++++++++++ 2 files changed, 264 insertions(+) create mode 100644 pkg/scriptmem/memory.go create mode 100644 pkg/scriptmem/memory_adapter_test.go diff --git a/pkg/scriptmem/memory.go b/pkg/scriptmem/memory.go new file mode 100644 index 0000000..32373ac --- /dev/null +++ b/pkg/scriptmem/memory.go @@ -0,0 +1,189 @@ +// Package script — memory.go is the bridge that attaches the existing +// in-memory runtime to the unified pipeline. A `memory` block resolves +// through the same Parse → Resolve front as everything else, then — instead +// of lowering to a Sibyl Plan (the temporal path) — is handed to the +// in-process interpreter that already implements every historical verb. +// +// Nothing about the runtime changes: it still consumes the old *Program +// AST and runs exactly as it always has. This file only ADAPTS the new +// resolved.AST into that *Program and calls the existing Execute. The two +// backends now share one front end (translate/parse/resolve) and diverge +// only at the final interpreter — temporal lowers+submits, memory adapts +// +executes — which is the whole point of the unification. +package scriptmem + +import ( + "context" + "fmt" + + "github.com/vinodhalaharvi/agentscript/internal/agentscript" + "github.com/vinodhalaharvi/agentscript/pkg/script/ast" + "github.com/vinodhalaharvi/agentscript/pkg/script/resolved" +) + +// MemoryConfig configures the in-memory runtime for a RunMemory call. It +// mirrors the runtime's own config; all fields are optional (a verb that +// needs a credential the config doesn't supply fails at execution, as it +// always has). +type MemoryConfig struct { + GeminiAPIKey string + ClaudeAPIKey string + SearchAPIKey string + Model string + Verbose bool + GoogleCredsFile string + GoogleTokenFile string + GitHubClientID string + GitHubClientSecret string + GitHubTokenFile string +} + +// RunMemory executes a resolved memory-backend program in-process and +// returns its result. It is the synchronous sibling of the temporal +// submit path: where temporal starts a durable workflow and returns a +// handle, RunMemory runs the interpreter here and now and returns the +// output directly. +// +// It expects the block to target the memory backend (callers route on the +// backend keyword); a non-memory block is rejected so a temporal program +// is never silently run in-process. +func RunMemory(ctx context.Context, cfg MemoryConfig, r resolved.AST) (string, error) { + prog, backend, err := toProgram(r) + if err != nil { + return "", err + } + if backend != ast.BackendMemory { + return "", fmt.Errorf("RunMemory: program targets the %s backend, not memory", backend.String()) + } + + rt, err := agentscript.NewRuntime(ctx, agentscript.RuntimeConfig{ + GeminiAPIKey: cfg.GeminiAPIKey, + ClaudeAPIKey: cfg.ClaudeAPIKey, + SearchAPIKey: cfg.SearchAPIKey, + Model: cfg.Model, + Verbose: cfg.Verbose, + GoogleCredsFile: cfg.GoogleCredsFile, + GoogleTokenFile: cfg.GoogleTokenFile, + GitHubClientID: cfg.GitHubClientID, + GitHubClientSecret: cfg.GitHubClientSecret, + GitHubTokenFile: cfg.GitHubTokenFile, + }) + if err != nil { + return "", fmt.Errorf("RunMemory: runtime init: %w", err) + } + return rt.Execute(ctx, prog) +} + +// toProgram adapts a resolved.AST into the in-memory runtime's *Program. +// The runtime is unchanged; this shim translates the unified pipeline's +// representation into the one Execute already understands. It returns the +// program and its backend so the caller can verify routing. +// +// The MVP pipeline produces exactly one block; multi-block programs are +// rejected (the temporal path has the same single-block assumption). +func toProgram(r resolved.AST) (*agentscript.Program, ast.Backend, error) { + if len(r.Blocks) != 1 { + return nil, 0, fmt.Errorf("toProgram: expected exactly one block, got %d", len(r.Blocks)) + } + b := r.Blocks[0] + stmt, err := nodeToStatement(b.Body) + if err != nil { + return nil, 0, err + } + return &agentscript.Program{Statements: []*agentscript.Statement{stmt}}, b.Backend, nil +} + +// nodeToStatement converts a resolved.Node into the old *Statement tree. +// - Pipeline → a chain of Statements linked by .Pipe (a >=> b >=> c) +// - Parallel → a Statement whose .Parallel holds the branches +// - Call → a Statement whose .Command holds the verb + args +func nodeToStatement(n resolved.Node) (*agentscript.Statement, error) { + switch node := n.(type) { + case resolved.Pipeline: + return pipelineToStatement(node) + case resolved.Parallel: + return parallelToStatement(node) + case resolved.Call: + cmd, err := callToCommand(node) + if err != nil { + return nil, err + } + return &agentscript.Statement{Command: cmd}, nil + default: + return nil, fmt.Errorf("nodeToStatement: unsupported node %T", n) + } +} + +// pipelineToStatement folds [s0, s1, s2] into s0 >=> s1 >=> s2 by linking +// each stage's .Pipe to the next. A single-stage pipeline is just that +// stage. +func pipelineToStatement(p resolved.Pipeline) (*agentscript.Statement, error) { + if len(p.Stages) == 0 { + return nil, fmt.Errorf("pipelineToStatement: empty pipeline") + } + stmts := make([]*agentscript.Statement, len(p.Stages)) + for i, s := range p.Stages { + st, err := nodeToStatement(s) + if err != nil { + return nil, err + } + stmts[i] = st + } + // Link i → i+1 via Pipe. A stage that is itself a pipeline/parallel + // statement keeps its own structure; we attach the continuation to + // the tail of the chain. + for i := 0; i < len(stmts)-1; i++ { + tail := stmts[i] + for tail.Pipe != nil { + tail = tail.Pipe + } + tail.Pipe = stmts[i+1] + } + return stmts[0], nil +} + +// parallelToStatement maps resolved.Parallel branches into the old +// *Parallel form ( b0 <*> b1 <*> ... ). +func parallelToStatement(p resolved.Parallel) (*agentscript.Statement, error) { + branches := make([]*agentscript.Statement, len(p.Branches)) + for i, br := range p.Branches { + st, err := nodeToStatement(br) + if err != nil { + return nil, err + } + branches[i] = st + } + return &agentscript.Statement{Parallel: &agentscript.Parallel{Branches: branches}}, nil +} + +// callToCommand maps a resolved.Call into the old *Command. The old form +// holds up to four positional string args; the pipeline's permissive +// memory specs are variadic-string, so we map the first four and reject +// anything beyond what the old command shape can carry. +func callToCommand(c resolved.Call) (*agentscript.Command, error) { + args := make([]string, 0, len(c.Args)) + for _, a := range c.Args { + s, ok := a.(ast.StringArg) + if !ok { + return nil, fmt.Errorf("callToCommand: %s: non-string argument %T", c.Name, a) + } + args = append(args, s.Value) + } + if len(args) > 4 { + return nil, fmt.Errorf("callToCommand: %s: %d args exceeds the 4 the in-memory command form supports", c.Name, len(args)) + } + cmd := &agentscript.Command{Action: c.Name} + if len(args) > 0 { + cmd.Arg = args[0] + } + if len(args) > 1 { + cmd.Arg2 = args[1] + } + if len(args) > 2 { + cmd.Arg3 = args[2] + } + if len(args) > 3 { + cmd.Arg4 = args[3] + } + return cmd, nil +} diff --git a/pkg/scriptmem/memory_adapter_test.go b/pkg/scriptmem/memory_adapter_test.go new file mode 100644 index 0000000..9480eb8 --- /dev/null +++ b/pkg/scriptmem/memory_adapter_test.go @@ -0,0 +1,75 @@ +package scriptmem + +import ( + "context" + "testing" + + "github.com/vinodhalaharvi/agentscript/internal/agentscript" + "github.com/vinodhalaharvi/agentscript/pkg/script" + "github.com/vinodhalaharvi/agentscript/pkg/script/ast" +) + +// Build a resolved memory program by running the real front end +// (script.Parse → script.Resolve) against the complete registry, then +// assert the adapter maps it to the correct old *Program shape. Exercises +// the bridge's structural translation without needing a live runtime. +func resolveMemory(t *testing.T, src string) (prog *agentscript.Program, backend ast.Backend) { + t.Helper() + a, err := script.Parse(context.Background(), script.Source(src)) + if err != nil { + t.Fatalf("parse: %v", err) + } + r, err := script.Resolve(context.Background(), script.CompleteRegistry(), a) + if err != nil { + t.Fatalf("resolve: %v", err) + } + prog, backend, err = toProgram(r) + if err != nil { + t.Fatalf("toProgram: %v", err) + } + return prog, backend +} + +func TestAdapter_SingleCall(t *testing.T) { + prog, backend := resolveMemory(t, `memory static ( hf_summarize "the thread" )`) + if backend != ast.BackendMemory { + t.Fatalf("backend = %v, want memory", backend) + } + if len(prog.Statements) != 1 { + t.Fatalf("statements = %d, want 1", len(prog.Statements)) + } + cmd := prog.Statements[0].Command + if cmd == nil { + t.Fatal("expected a Command") + } + if cmd.Action != "hf_summarize" { + t.Errorf("Action = %q, want hf_summarize", cmd.Action) + } + if cmd.Arg != "the thread" { + t.Errorf("Arg = %q, want 'the thread'", cmd.Arg) + } +} + +func TestAdapter_PipelineChains(t *testing.T) { + prog, _ := resolveMemory(t, `memory static ( search "x" >=> hf_summarize >=> echo )`) + s := prog.Statements[0] + names := []string{} + for s != nil { + if s.Command != nil { + names = append(names, s.Command.Action) + } + s = s.Pipe + } + if len(names) != 3 || names[0] != "search" || names[1] != "hf_summarize" || names[2] != "echo" { + t.Errorf("pipeline chain = %v, want [search hf_summarize echo]", names) + } +} + +func TestAdapter_RejectsTemporalBackend(t *testing.T) { + a, _ := script.Parse(context.Background(), script.Source(`temporal static ( echo "x" )`)) + r, _ := script.Resolve(context.Background(), script.CompleteRegistry(), a) + _, err := RunMemory(context.Background(), MemoryConfig{}, r) + if err == nil { + t.Fatal("RunMemory should reject a temporal-backend program") + } +}