Implement persistent backup to/ restore from zip files

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ jobs:
             wasm-build/output/pglite.wasi
             core/src/main/resources/pgdata/
             core/src/main/resources/pglite-files.txt
+            core/src/main/resources/pglite-dirs.txt
 
   build:
     needs: wasm-build

.gitignore

@@ -7,3 +7,6 @@ core/src/main/resources/pglite/
 core/src/main/resources/pgdata/
 core/src/main/resources/*
 .idea
+
+# Backup zip files created by tests
+memory:*

README.md

@@ -53,39 +53,60 @@ Add the JDBC driver dependency:
 ### Plain JDBC
 
 ```java
+// In-memory (ephemeral) — data is lost when the JVM exits
 Connection conn = DriverManager.getConnection("jdbc:pglite:memory://");
 conn.createStatement().execute("CREATE TABLE demo (id serial PRIMARY KEY, name text)");
 conn.createStatement().execute("INSERT INTO demo (name) VALUES ('hello')");
 ```
 
+### Persistent storage
+
+Point the JDBC URL to a file path and `pglite4j` will periodically snapshot the entire in-memory database to a zip file on disk. On the next JVM startup, the database is restored from that snapshot.
+
+> **Note:** This is **not** traditional disk-backed storage. PostgreSQL runs entirely in memory (ZeroFS). The driver takes periodic snapshots (backup/restore), similar to Redis RDB persistence. Data written between the last snapshot and a crash will be lost. This is suitable for demo apps, prototyping, and development — not for production workloads that require durability guarantees.
+
+```java
+// File-backed — data survives JVM restarts
+Connection conn = DriverManager.getConnection("jdbc:pglite:/var/data/mydb.zip");
+```
+
+The driver backs up the database on a fixed schedule (default: every 60 seconds) and writes a final snapshot on shutdown. You can configure the backup interval via a connection property:
+
+```java
+Properties props = new Properties();
+props.setProperty("pgliteBackupIntervalSeconds", "30");
+Connection conn = DriverManager.getConnection("jdbc:pglite:/var/data/mydb.zip", props);
+```
+
+You can also use named in-memory databases for test isolation (separate PG instances, no persistence):
+
+```java
+Connection db1 = DriverManager.getConnection("jdbc:pglite:memory:testA");
+Connection db2 = DriverManager.getConnection("jdbc:pglite:memory:testB");
+```
+
 ### Quarkus
 
 ```properties
 # application.properties
 quarkus.datasource.db-kind=postgresql
 quarkus.datasource.jdbc.url=jdbc:pglite:memory://
+# or persistent: jdbc:pglite:/var/data/myapp.zip
 quarkus.datasource.jdbc.driver=io.roastedroot.pglite4j.jdbc.PgLiteDriver
-quarkus.datasource.username=postgres
-quarkus.datasource.password=password
-quarkus.datasource.jdbc.min-size=1
 quarkus.datasource.jdbc.max-size=5
 quarkus.devservices.enabled=false
-quarkus.hibernate-orm.dialect=org.hibernate.dialect.PostgreSQLDialect
-quarkus.hibernate-orm.unsupported-properties."hibernate.boot.allow_jdbc_metadata_access"=false
 ```
 
 ### Spring Boot
 
 ```properties
-# application-test.properties
+# application.properties
 spring.datasource.url=jdbc:pglite:memory://
+# or persistent: jdbc:pglite:/var/data/myapp.zip
 spring.datasource.driver-class-name=io.roastedroot.pglite4j.jdbc.PgLiteDriver
-spring.datasource.username=postgres
-spring.datasource.password=password
 spring.datasource.hikari.maximum-pool-size=5
 spring.jpa.hibernate.ddl-auto=create-drop
 spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
-spring.jpa.properties.hibernate.boot.allow_jdbc_metadata_access=false
 ```
 
 ### HikariCP
@@ -109,7 +130,7 @@ pglite4j/
 
 ## Status and known limitations
 
-- [x] ~~**Only `memory://` is supported**~~ — persistent / file-backed databases are not planned; the WASM backend uses an in-memory virtual filesystem (ZeroFS) with no disk I/O, which is fundamental to the architecture
+- [x] ~~**Only `memory://` is supported**~~ — file-backed storage is now supported via periodic snapshots. The database runs entirely in memory; the driver takes a full snapshot (zip of pgdata) on a configurable schedule and on shutdown. On restart the snapshot is restored. This is backup/restore-style persistence (like Redis RDB), not write-ahead logging — data between the last snapshot and a crash is lost
 - [x] ~~**Single connection only**~~ — multiple JDBC connections are now supported per database instance; requests are serialized through a single PGLite backend via a lock, so connection pools with `max-size > 1` work correctly (queries execute one at a time, not in parallel)
 - [x] ~~**Error recovery**~~ — both simple and extended query protocol errors are handled correctly; PostgreSQL errors trap the WASM instance and are caught by the Java side, which resets the backend state and drains stale protocol buffers so subsequent queries work cleanly
 - [ ] **No connection isolation** — PostgreSQL runs in single-user mode with one session; all connections share the same session state (transactions, session variables). Queries are serialized, so there is no data corruption, but concurrent transactions are not isolated from each other. This is fine for connection pools that use connections sequentially (borrow, use, return).

core/src/main/java/io/roastedroot/pglite4j/core/PGLite.java

@@ -12,11 +12,17 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
 
 @WasmModuleInterface(WasmResource.absoluteFile)
 public final class PGLite implements AutoCloseable {
@@ -30,10 +36,12 @@ public final class PGLite implements AutoCloseable {
     private final WasiPreview1 wasi;
     private final PGLite_ModuleExports exports;
     private final FileSystem fs;
+    private final Path dataDir;
     private int bufferAddr;
     private int pendingWireLen;
 
-    private PGLite() {
+    private PGLite(Path dataDir) {
+        this.dataDir = dataDir;
         try {
             this.fs =
                     ZeroFs.newFileSystem(
@@ -42,6 +50,12 @@ private PGLite() {
             // Extract pgdata files into ZeroFS.
             // (share + lib are embedded in the WASM binary via wasi-vfs)
             extractDistToZeroFs(fs);
+
+            // Restore saved pgdata from a previous session (overwrites defaults).
+            if (dataDir != null && java.nio.file.Files.exists(dataDir)) {
+                restoreDataDir(dataDir);
+            }
+
             Path tmp = fs.getPath("/tmp");
             Files.createDirectories(tmp);
             Path pgdata = fs.getPath("/pgdata");
@@ -89,8 +103,15 @@ private PGLite() {
                             .build();
             this.exports = new PGLite_ModuleExports(this.instance);
 
-            // pgl_initdb + pgl_backend already executed by wizer at build time.
-            // closeAllVfds() was called at end of wizer to prevent stale fd PANICs.
+            if (dataDir != null && java.nio.file.Files.exists(dataDir)) {
+                // Restored pgdata differs from the wizer snapshot.
+                // The wizer snapshot's shared buffer pool has stale catalog
+                // pages from the clean template1 database; invalidate them
+                // so PostgreSQL re-reads from the restored ZeroFS files.
+                exports.pglInvalidateBuffers();
+            }
+
+            // pgl_initdb + pgl_backend already executed (by wizer or restart above).
             exports.interactiveWrite(0);
 
             int channel = exports.getChannel();
@@ -149,8 +170,61 @@ public static Builder builder() {
         return new Builder();
     }
 
+    /**
+     * Snapshot the in-memory pgdata directory to a zip file on the host filesystem.
+     * Runs VACUUM FREEZE + CHECKPOINT first so that all tuples are frozen (visible
+     * after restore without CLOG) and all dirty buffers are flushed to ZeroFS files.
+     * Writes atomically via temp file + move.
+     */
+    public void dumpDataDir(Path target) throws IOException {
+        // Freeze all tuple xmin values so they survive restore without
+        // needing the CLOG (commit log) cache from the original session.
+        // Then CHECKPOINT to flush all dirty buffers to ZeroFS files.
+        execProtocolRaw(buildSimpleQuery("VACUUM FREEZE;"));
+        execProtocolRaw(buildSimpleQuery("CHECKPOINT;"));
+
+        Path tmp = target.resolveSibling(target.getFileName() + ".tmp");
+        Path pgdataRoot = fs.getPath(PG_DATA);
+
+        try (ZipOutputStream zos = new ZipOutputStream(java.nio.file.Files.newOutputStream(tmp))) {
+            try (Stream<Path> walk = Files.walk(pgdataRoot)) {
+                walk.filter(Files::isRegularFile)
+                        .filter(p -> !p.getFileName().toString().startsWith(".s.PGSQL."))
+                        .forEach(
+                                p -> {
+                                    try {
+                                        String entryName = pgdataRoot.relativize(p).toString();
+                                        zos.putNextEntry(new ZipEntry(entryName));
+                                        Files.copy(p, zos);
+                                        zos.closeEntry();
+                                    } catch (IOException e) {
+                                        throw new RuntimeException("Failed to zip " + p, e);
+                                    }
+                                });
+            }
+        }
+
+        // Atomic move; fall back to plain replace if the filesystem doesn't support it.
+        try {
+            java.nio.file.Files.move(
+                    tmp,
+                    target,
+                    StandardCopyOption.ATOMIC_MOVE,
+                    StandardCopyOption.REPLACE_EXISTING);
+        } catch (AtomicMoveNotSupportedException e) {
+            java.nio.file.Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING);
+        }
+    }
+
     @Override
     public void close() {
+        if (dataDir != null) {
+            try {
+                dumpDataDir(dataDir);
+            } catch (IOException | RuntimeException e) {
+                // best-effort backup on close
+            }
+        }
         try {
             exports.pglShutdown();
         } catch (RuntimeException e) {
@@ -224,6 +298,19 @@ private boolean collectReply(List<byte[]> replies) {
         return false;
     }
 
+    private static byte[] buildSimpleQuery(String query) {
+        byte[] sql = query.getBytes(StandardCharsets.UTF_8);
+        byte[] msg = new byte[1 + 4 + sql.length + 1];
+        msg[0] = 'Q';
+        int len = 4 + sql.length + 1;
+        msg[1] = (byte) (len >> 24);
+        msg[2] = (byte) (len >> 16);
+        msg[3] = (byte) (len >> 8);
+        msg[4] = (byte) len;
+        System.arraycopy(sql, 0, msg, 5, sql.length);
+        return msg;
+    }
+
     private static byte[] concat(List<byte[]> replies) {
         int totalLen = 0;
         for (byte[] r : replies) {
@@ -238,8 +325,43 @@ private static byte[] concat(List<byte[]> replies) {
         return result;
     }
 
+    private void restoreDataDir(Path source) throws IOException {
+        Path pgdataRoot = fs.getPath(PG_DATA);
+        try (ZipInputStream zis = new ZipInputStream(java.nio.file.Files.newInputStream(source))) {
+            ZipEntry entry;
+            while ((entry = zis.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    continue;
+                }
+                Path target = pgdataRoot.resolve(entry.getName());
+                Files.createDirectories(target.getParent());
+                // Overwrite classpath defaults with saved state.
+                Files.deleteIfExists(target);
+                Files.copy(zis, target);
+            }
+        }
+    }
+
     // === Resource extraction ===
     private static void extractDistToZeroFs(FileSystem fs) throws IOException {
+        // Create all pgdata directories first (including empty ones that
+        // PostgreSQL expects, e.g. pg_logical/snapshots).
+        InputStream dirManifest = PGLite.class.getResourceAsStream("/pglite-dirs.txt");
+        if (dirManifest != null) {
+            try (BufferedReader dr =
+                    new BufferedReader(
+                            new InputStreamReader(dirManifest, StandardCharsets.UTF_8))) {
+                String line;
+                while ((line = dr.readLine()) != null) {
+                    line = line.trim();
+                    if (line.isEmpty()) {
+                        continue;
+                    }
+                    Files.createDirectories(fs.getPath("/" + line));
+                }
+            }
+        }
+
         InputStream manifest = PGLite.class.getResourceAsStream("/pglite-files.txt");
         if (manifest == null) {
             throw new RuntimeException(
@@ -267,10 +389,18 @@ private static void extractDistToZeroFs(FileSystem fs) throws IOException {
     }
 
     public static final class Builder {
+        private Path dataDir;
+
         private Builder() {}
 
+        /** Set a host filesystem path for pgdata backup/restore (zip file). */
+        public Builder withDataDir(Path dataDir) {
+            this.dataDir = dataDir;
+            return this;
+        }
+
         public PGLite build() {
-            return new PGLite();
+            return new PGLite(dataDir);
         }
     }
 }

core/src/test/java/io/roastedroot/pglite4j/core/PGLiteTest.java

@@ -3,6 +3,8 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import org.junit.jupiter.api.Test;
 
 public class PGLiteTest {
@@ -134,6 +136,84 @@ public void extendedProtocolErrorRecovery() {
         }
     }
 
+    @Test
+    public void dumpCreatesValidZip() throws Exception {
+        Path backupFile = Files.createTempFile("pglite-backup-", ".zip");
+        Files.delete(backupFile);
+
+        try {
+            try (PGLite pg = PGLite.builder().build()) {
+                doHandshake(pg);
+
+                pg.execProtocolRaw(
+                        PgWireCodec.queryMessage("CREATE TABLE persist_test (id INT, val TEXT);"));
+                pg.execProtocolRaw(
+                        PgWireCodec.queryMessage(
+                                "INSERT INTO persist_test VALUES (1, 'survived');"));
+
+                pg.dumpDataDir(backupFile);
+                assertTrue(Files.exists(backupFile), "Backup file should exist after dump");
+                assertTrue(Files.size(backupFile) > 0, "Backup file should not be empty");
+
+                // Verify the zip contains pgdata files.
+                java.util.Set<String> entries = new java.util.HashSet<>();
+                try (java.util.zip.ZipInputStream zis =
+                        new java.util.zip.ZipInputStream(Files.newInputStream(backupFile))) {
+                    java.util.zip.ZipEntry e;
+                    while ((e = zis.getNextEntry()) != null) {
+                        entries.add(e.getName());
+                    }
+                }
+                assertTrue(entries.contains("PG_VERSION"), "Zip should contain PG_VERSION");
+                assertTrue(
+                        entries.stream().anyMatch(n -> n.startsWith("base/")),
+                        "Zip should contain base/ directory entries");
+            }
+        } finally {
+            Files.deleteIfExists(backupFile);
+        }
+    }
+
+    @Test
+    public void dataDirectoryBackupRestore() throws Exception {
+        Path backupFile = Files.createTempFile("pglite-backup-", ".zip");
+        Files.delete(backupFile); // start without an existing backup
+
+        try {
+            // Session 1: create data and dump.
+            try (PGLite pg = PGLite.builder().withDataDir(backupFile).build()) {
+                doHandshake(pg);
+
+                pg.execProtocolRaw(
+                        PgWireCodec.queryMessage("CREATE TABLE persist_test (id INT, val TEXT);"));
+                pg.execProtocolRaw(
+                        PgWireCodec.queryMessage(
+                                "INSERT INTO persist_test VALUES (1, 'survived');"));
+
+                pg.dumpDataDir(backupFile);
+                assertTrue(Files.exists(backupFile), "Backup file should exist after dump");
+                assertTrue(Files.size(backupFile) > 0, "Backup file should not be empty");
+            }
+
+            // Session 2: restore from the dump and verify data survived.
+            try (PGLite pg = PGLite.builder().withDataDir(backupFile).build()) {
+                doHandshake(pg);
+
+                byte[] r =
+                        pg.execProtocolRaw(
+                                PgWireCodec.queryMessage(
+                                        "SELECT val FROM persist_test WHERE id = 1;"));
+                String data = PgWireCodec.parseDataRows(r);
+                assertTrue(
+                        data.contains("survived"),
+                        "Data should survive restart via backup/restore, got: " + data);
+            }
+        } finally {
+            Files.deleteIfExists(backupFile);
+            Files.deleteIfExists(backupFile.resolveSibling(backupFile.getFileName() + ".tmp"));
+        }
+    }
+
     static void doHandshake(PGLite pg) {
         byte[] startup = PgWireCodec.startupMessage("postgres", "template1");
         byte[] resp1 = pg.execProtocolRaw(startup);