Persistent databases

.github/workflows/main.yml

@@ -53,4 +53,4 @@ jobs:
           key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile.lock') }}
           restore-keys: ${{ runner.os }}-gems-
       - name: Run the tests
-        run: devenv tasks run test
+        run: devenv test

.mutant.yml

@@ -33,3 +33,4 @@ matcher:
     - En57::Benchmark::ResConcurrentAppendConflictingStreams*
     - En57::Benchmark::ResConcurrentAppendNonConflictingStreams*
     - En57::Benchmark::Scenario#concurrently # thread raise mutation survives despite direct coverage
+    - En57::EphemeralDatabase#execute # receiver->self mutation runs Kernel#exec on multi-word SQL, replacing the worker

AGENTS.md

@@ -1,5 +1,5 @@
 # Agent rules
 
-- Prefer inlining variables when they are not reused.
-- After editing files, run `treefmt`.
-- When adding and building a feature, follow the TDD process.
+- Prefer inlining variables when they are not reused
+- When adding and building a feature, follow the TDD process
+- Never add code comments, keep method and variable names clear enough for comments not to be needed, keep "the why" comments in commit messages

Gemfile

@@ -14,8 +14,6 @@ gem "minitest-mock"
 gem "minitest-stub-const"
 gem "mutant-minitest"
 gem "concurrent-ruby"
-gem "pg-ephemeral"
-gem "syntax_tree"
 
 gem "activerecord"
 gem "sequel"

Gemfile.lock

@@ -160,19 +160,11 @@ GEM
     parser (3.3.11.1)
       ast (~> 2.4.1)
       racc
-    pg (1.6.3)
     pg (1.6.3-aarch64-linux)
     pg (1.6.3-arm64-darwin)
     pg (1.6.3-x86_64-linux)
-    pg-ephemeral (0.5.1-aarch64-linux)
-      pg (~> 1.5)
-    pg-ephemeral (0.5.1-arm64-darwin)
-      pg (~> 1.5)
-    pg-ephemeral (0.5.1-x86_64-linux)
-      pg (~> 1.5)
     pp (0.6.3)
       prettyprint
-    prettier_print (1.2.1)
     prettyprint (0.2.0)
     prism (1.9.0)
     psych (5.4.0)
@@ -249,8 +241,6 @@ GEM
       bigdecimal
     sorbet-runtime (0.6.13270)
     stringio (3.2.0)
-    syntax_tree (6.3.0)
-      prettier_print (>= 1.2.0)
     thor (1.5.0)
     timeout (0.6.1)
     tsort (0.2.0)
@@ -286,12 +276,10 @@ DEPENDENCIES
   minitest-mock
   minitest-stub-const
   mutant-minitest
-  pg-ephemeral
   rails
   rails_event_store
   rake
   sequel
-  syntax_tree
 
 CHECKSUMS
   action_text-trix (2.1.19) sha256=7012f59421009cf284aa651294896414d653a61a2417c9b8714c8476d2f74009
@@ -348,15 +336,10 @@ CHECKSUMS
   nokogiri (1.19.3-arm64-darwin) sha256=71b9bd424b1b7abc18b05052a1a3cfd3627abdca62be280854cc411791357e42
   nokogiri (1.19.3-x86_64-linux-gnu) sha256=2f5078620fe12e83669b5b17311b32532a8153d02eee7ad06948b926d6080976
   parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54
-  pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99
   pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea
   pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f
   pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d
-  pg-ephemeral (0.5.1-aarch64-linux) sha256=109ede089c1a5ef2d5ef80e8a1a8d7e4741a827948a1e7c3ae44c37f27fe6dbe
-  pg-ephemeral (0.5.1-arm64-darwin) sha256=b254bacea36f1fd9b9690f95d141c16716684beccae7ffd0a23df4477b4cf512
-  pg-ephemeral (0.5.1-x86_64-linux) sha256=4c7a867df7e88abd5d8a5e816b0f633043c181017ccc6cde90b4be2087361129
   pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
-  prettier_print (1.2.1) sha256=a72838b5f23facff21f90a5423cdcdda19e4271092b41f4ea7f50b83929e6ff9
   prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
   psych (5.4.0) sha256=14f72d69a611af663d7d70e4a7b67d9eb1f3ae9f8d916b478961d5a0075ba5b7
@@ -382,7 +365,6 @@ CHECKSUMS
   sequel (5.105.0) sha256=4191d5e5d011b7229ff8a3401ca6beac047c4d35b1ec6703376513b4b25b2e67
   sorbet-runtime (0.6.13270) sha256=e7bc6899e438e8fa02fb9324febb4433538b262ecfe8ce770cbd1221e2bf3276
   stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
-  syntax_tree (6.3.0) sha256=56e25a9692c798ec94c5442fe94c5e94af76bef91edc8bb02052cbdecf35f13d
   thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73
   timeout (0.6.1) sha256=78f57368a7e7bbadec56971f78a3f5ecbcfb59b7fcbb0a3ed6ddc08a5094accb
   tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f

README.md

@@ -182,18 +182,36 @@ devenv shell
 This provides Ruby, PostgreSQL, and the formatters, and installs the gem
 dependencies (`bundle install`, via the `dev:setup` task) on entry.
 
+`devenv shell` provides the toolchain but does not start the PostgreSQL
+service. Bring the services up before running anything that touches the
+database:
+
+```sh
+devenv up      # foreground; Ctrl-C to stop
+devenv up -d   # detached; stop later with `devenv processes down`
+```
+
 Tasks are run with `devenv tasks run`:
 
 | Task | What it does |
 |------|--------------|
 | `test` | Run the whole `test:` namespace (unit, mutation, pg_regress) |
 | `test:unit` | Run the unit tests (`bin/m test`) |
 | `test:mutate` | Run mutation testing (`mutant`) for changes since `MUTANT_SINCE` (defaults to `HEAD`) |
-| `test:pg` | Run the `pg_regress` suite against an ephemeral PostgreSQL |
+| `test:pg` | Run the `pg_regress` suite against the running PostgreSQL service |
 | `dev:format` | Format Ruby and SQL with [treefmt](https://treefmt.com) (syntax_tree + sqlfluff) |
 
+`devenv tasks run` does not start services, so the DB-backed tests assume
+PostgreSQL is already up (see `devenv up` above). To start the services,
+wait for them to be healthy, run the suite, and tear them down in one step,
+use:
+
+```sh
+devenv test
+```
+
 `pg-regress` is also available as a standalone script in the shell.
 
-CI runs `devenv tasks run test`. The devenv config also wires up Claude Code
-hooks: edited files are formatted with treefmt, and the test suite runs at the
-end of each agent loop.
+CI runs `devenv test`. The devenv config also wires up Claude Code hooks:
+edited files are formatted with treefmt, and the test suite runs at the end
+of each agent loop.

devenv.nix

@@ -22,6 +22,7 @@
     pkgs.postgresql_18
     pkgs.postgresql_18.pg_config
     pkgs.sqlfluff
+    pkgs.rubyPackages.syntax_tree
     pkgs.jq
     pkgs.libyaml
     pkgs.pi-coding-agent
@@ -33,6 +34,36 @@
     export MUTANT_SINCE="''${MUTANT_SINCE:-HEAD}"
   '';
 
+  enterTest = ''
+    devenv tasks run test
+  '';
+
+  services.postgres = {
+    enable = true;
+    package = pkgs.postgresql_18;
+    initialDatabases =
+      let
+        seededSchema = pkgs.runCommand "golden-en57-seeded.sql" { } ''
+          cat ${./db/schema/0.1.0.sql} \
+              ${./db/seeds/concurrent_append_non_conflicting_tags_seeded.sql} > $out
+        '';
+      in
+      [
+        {
+          name = "golden_en57";
+          schema = ./db/schema/0.1.0.sql;
+        }
+        {
+          name = "golden_res";
+          schema = ./db/seeds/res.sql;
+        }
+        {
+          name = "golden_en57_seeded";
+          schema = seededSchema;
+        }
+      ];
+  };
+
   files =
     let
       commitSkill = ''
@@ -89,7 +120,7 @@
           "*" = "treefmt {file}";
         };
         onStop = {
-          "*" = "devenv tasks run test";
+          "*" = "devenv test";
         };
       };
     };
@@ -135,7 +166,7 @@
     };
     settings.formatter = {
       ruby = {
-        command = "stree";
+        command = "${pkgs.rubyPackages.syntax_tree}/bin/stree";
         options = [ "write" ];
         includes = [ "*.rb" ];
       };
@@ -159,32 +190,21 @@
 
     pg_regress=${pkgs.postgresql_18.dev}/lib/pgxs/src/test/regress/pg_regress
     bindir=${pkgs.postgresql_18}/bin
-    workdir=$(mktemp -d)
-    datadir=$workdir/data
-    socketdir=$workdir/socket
-    dbname=en57_regress
-    mkdir -p "$socketdir"
-
-    cleanup() {
-      "$bindir/pg_ctl" -D "$datadir" -m immediate stop >/dev/null 2>&1 || true
-      rm -rf "$workdir"
-    }
-    trap cleanup EXIT
-
-    "$bindir/initdb" -D "$datadir" -U postgres --auth=trust >/dev/null
-    "$bindir/pg_ctl" -D "$datadir" -w \
-      -o "-k $socketdir -c listen_addresses='''" start >/dev/null
-    "$bindir/createdb" -h "$socketdir" -U postgres "$dbname"
-    "$bindir/psql" -h "$socketdir" -U postgres -d "$dbname" \
+    dbname=regress
+    user=''${PGUSER:-$(id -un)}
+
+    "$bindir/dropdb" -h "$PGHOST" -U "$user" --if-exists "$dbname"
+    "$bindir/createdb" -h "$PGHOST" -U "$user" "$dbname"
+    "$bindir/psql" -h "$PGHOST" -U "$user" -d "$dbname" \
       -v ON_ERROR_STOP=1 -q -f db/schema/0.1.0.sql
 
     rm -rf test/pg_regress/results
     mkdir -p test/pg_regress/results
 
     "$pg_regress" \
       --use-existing \
-      --host="$socketdir" \
-      --user=postgres \
+      --host="$PGHOST" \
+      --user="$user" \
       --dbname="$dbname" \
       --inputdir=test/pg_regress \
       --outputdir=test/pg_regress/results \
@@ -204,12 +224,12 @@
       '';
     };
     test = {
-      name = "Run the test namespace on stop";
+      name = "Run devenv test on stop";
       hookType = "Stop";
       command = ''
         input=$(cat)
         cd "''${DEVENV_ROOT:-.}" || exit 0
-        devenv tasks run test 1>&2 && exit 0
+        devenv test 1>&2 && exit 0
         [ "$(printf '%s' "$input" | jq -r '.stop_hook_active // false')" = "true" ] && exit 0
         exit 2
       '';

lib/benchmark/concurrent_append_non_conflicting_tags.rb

@@ -38,6 +38,7 @@ def call(measure, retries, _run_id)
     concurrent_append_non_conflicting_tags.with(
       database_instance: "concurrent-append-non-conflicting-tags-seeded",
       name: "10x100 concurrent append, non-conflicting tags (seeded)",
+      template: "golden_en57_seeded",
     )
   end
 end

lib/benchmark/res_append_stream_any.rb

@@ -8,6 +8,7 @@ module Benchmark
       runs: ->(runs) { runs * 10 },
       concurrency: 1,
       batch_size: 100,
+      template: "golden_res",
     ) do
       def setup(database_url)
         require "active_record"

lib/benchmark/res_concurrent_append_conflicting_streams.rb

@@ -7,6 +7,7 @@ module Benchmark
       name: "10x100 concurrent append, conflicting streams (RES)",
       concurrency: 10,
       batch_size: 100,
+      template: "golden_res",
     ) do
       def setup(database_url)
         require "active_record"

lib/benchmark/res_concurrent_append_non_conflicting_streams.rb

@@ -7,6 +7,7 @@ module Benchmark
       name: "10x100 concurrent append, non-conflicting streams (RES)",
       concurrency: 10,
       batch_size: 100,
+      template: "golden_res",
     ) do
       def setup(database_url)
         require "active_record"