Make SQLite the default local/Docker database

.env.quickstart.example

@@ -50,30 +50,43 @@ AUTH_MODE=local
 
 # DATABASE_TYPE
 #   What:  Which persistence backend to use for state and session data.
-#          "emulator" = bundled Firestore emulator container (no GCP needed).
-#          "firestore" = real GCP Firestore (production).
+#          "sqlite" = local file, no GCP needed, no separate container (default).
+#          "firestore" = connect to an already-hosted, GCP-managed Firestore
+#                        instance (production, or your own hosted instance).
+#          "emulator" = connect to a manually-run Firestore emulator process
+#                       (SpecFlow does not start one for you).
 #          "memory" = in-memory only (unit tests).
-#   How:   Leave as "emulator" for quickstart. No GCP credentials required.
-DATABASE_TYPE=emulator
+#   How:   Leave as "sqlite" for quickstart. No GCP credentials required.
+DATABASE_TYPE=sqlite
+
+# SPECFLOW_HOME_MOUNT_PATH
+#   What:  Host-side directory bind-mounted into the backend container at
+#          /root/.specflow (the container's SpecFlow home; the SQLite db lives at
+#          db/specflow.db inside it — one central database shared across every local
+#          project/MCP session on this machine, the way the Firestore emulator used
+#          to be one shared local instance). Must be on real
+#          (block) storage, never NFS/Filestore — SQLite's file locking is
+#          unsafe over NFS. Docker creates this host directory automatically
+#          on first `docker compose up` if it doesn't already exist.
+#   How:   Leave unset to default to ~/.specflow.
+# SPECFLOW_HOME_MOUNT_PATH=
 
 # FIRESTORE_EMULATOR_HOST
-#   What:  Network address of the Firestore emulator. Only used when
-#          DATABASE_TYPE=emulator.
-#   How:   Leave as "localhost:8080". Host-side scripts (init_firestore.py)
-#          connect here. Inside the backend container, docker-compose
-#          automatically injects "firestore-emulator:8080".
+#   What:  Network address of a manually-run Firestore emulator. Only used
+#          when DATABASE_TYPE=emulator.
+#   How:   Leave as "localhost:8080" if you run one yourself; otherwise unused.
 FIRESTORE_EMULATOR_HOST=localhost:8080
 
 # GCP_PROJECT_ID
-#   What:  GCP project identifier passed to the Firestore client.
-#          For the emulator this is an arbitrary string (not validated).
-#          For production Firestore, use your real GCP project ID.
-#   How:   Leave as "local-dev" for quickstart.
+#   What:  GCP project identifier passed to the Firestore client. Only used
+#          when DATABASE_TYPE=firestore or emulator.
+#   How:   Leave as "local-dev" for quickstart (sqlite ignores this).
 GCP_PROJECT_ID=local-dev
 
 # FIRESTORE_DATABASE_NAME
-#   What:  Logical database name within the Firestore project.
-#   How:   Leave as "specflow" for quickstart.
+#   What:  Logical database name within the Firestore project. Only used
+#          when DATABASE_TYPE=firestore or emulator.
+#   How:   Leave as "specflow" for quickstart (sqlite ignores this).
 FIRESTORE_DATABASE_NAME=specflow
 
 # WORKSPACE_BASE_PATH / AGENT_LOGS_BASE_PATH
@@ -84,13 +97,6 @@ FIRESTORE_DATABASE_NAME=specflow
 WORKSPACE_BASE_PATH=/workspaces
 AGENT_LOGS_BASE_PATH=/agent_logs
 
-# FIRESTORE_EXPORT_INTERVAL_SECONDS
-#   What:  How often (in seconds) the emulator sidecar exports Firestore state
-#          to ./workspaces/firestore_emulator/ for crash durability. On restart
-#          the emulator restores from the last export automatically.
-#   How:   Leave as 25. Reset all exported data with: ./specflow-init.sh --reset-local-db
-FIRESTORE_EXPORT_INTERVAL_SECONDS=25
-
 # USER_EMAIL / GIT_USER_EMAIL
 #   What:  Your email identity used for MCP config, notification emails, and
 #          git commit authorship on workspaces.

QUICKSTART.md

@@ -178,12 +178,19 @@ Stop the local stack:
 docker compose down --timeout 90
 ```
 
-Reset local Firestore state and reseed:
+Reset the local SQLite database and reseed:
 
 ```bash
 specflow init --reset-local-db
 ```
 
+Inspect the database: install [DB Browser for SQLite](https://sqlitebrowser.org/)
+(`brew install --cask db-browser-for-sqlite`), then open it read-only against the live db:
+
+```bash
+open -a "DB Browser for SQLite" ~/.specflow/db/specflow.db
+```
+
 ## More Guides
 
 - [Compass/P10Y setup](docs/quickstart-compass.md)

agents/MEMORY.md

@@ -8,6 +8,6 @@ Concise, generalized lessons (not a changelog — that is `agents/IMPLEMENTATION
 - For first-run `run_generation`, validate upload contract before creating generation sessions or allocating workspace sets; once `/workspace/sync` returns a `generation_id`, MCP may persist `specflow_session.json`.
 - When fixing a rejection/validation bug, enumerate every rejection code and every path that raises that rejection before planning the side-effect boundary; do not plan from only the observed failure case.
 - Firestore emulator imports require the `*overall_export_metadata` file path; export destinations are directories, but `--import-data` must not point at the snapshot directory itself.
-- Global SpecFlow/TUI settings live in `~/.specflow/config.json` (SSOT) — read/write via `mcp_server/tui/mcp_clients.py` `_read_config`/`_write_config`, which preserve unknown top-level keys; add each new setting as its own top-level section. Do NOT put global settings in the project-local `.specflow-local/` (that dir is per-project runtime: `mcp-config.json`, `workspaces.json`, `init.log`). MCP-client connection status is stored globally because connecting a client is a machine-wide act (`claude/gemini mcp add -s user`, Cursor `~/.cursor/mcp.json`).
+- Global SpecFlow/TUI settings live in `~/.specflow/config.json` (SSOT) — read/write via `mcp_server/tui/mcp_clients.py` `_read_config`/`_write_config`, which preserve unknown top-level keys; add each new setting as its own top-level section. Do NOT put global settings in the project-local `.specflow-local/` (that dir is per-project runtime: `mcp-config.json`, `init.log`; the workspace pool now lives only in the SQLite DB, seeded straight in by `create_generation_session_repos.py` — no `workspaces.json` flat file). MCP-client connection status is stored globally because connecting a client is a machine-wide act (`claude/gemini mcp add -s user`, Cursor `~/.cursor/mcp.json`).
 - Run MCP server pytest commands from `mcp_server/` (`uv run pytest tests/...`); root-level `uv run pytest ...` may not expose a pytest executable because the repo has per-package `pyproject.toml` environments.
 - P10Y repository list reads in Quickstart must use the narrowest available search (`github_org/prefix` for generated repos) and paginate; first-page org scans miss repos in large Compass organizations.

backend/Dockerfile

@@ -104,6 +104,8 @@ RUN uv sync --frozen --no-dev || uv sync --no-dev && \
 
 # Copy application code (preserve the app directory structure)
 COPY app ./app
+# scripts/ (init_db.py etc.) — needed at runtime via `docker compose exec backend`.
+COPY scripts ./scripts
 
 # Create .claude directory and copy settings to root's home directory
 RUN mkdir -p /root/.claude

backend/app/core/app_lifecycle.py

@@ -28,6 +28,7 @@
 )
 from app.core.logging import _cleanup_queue_logging
 from app.database.factory import get_database
+from app.database.sqlite import SqliteDatabase
 from app.jobs.shutdown_interrupted_recovery import recover_interrupted_sessions
 from app.jobs.stuck_cleaning_recovery import recover_stuck_cleaning
 from app.jobs.stuck_initializing_detector import detect_stuck_initializing
@@ -188,6 +189,10 @@ async def lifespan(app: FastAPI):
 
     await run_shutdown_session_handling(db)
 
+    if isinstance(raw_db, SqliteDatabase):
+        logger.info("Closing SQLite connection before shutdown...")
+        raw_db.close()
+
     if not validation_task.done():
         logger.info("Cancelling startup validation task...")
         validation_task.cancel()

backend/app/core/config.py

@@ -23,6 +23,12 @@
 WORKSPACE_DEFAULT_BRANCH = "main"
 WORKSPACE_DEPLOY_WORKFLOW = "deploy.yml"
 
+# Fixed container-internal path to the SQLite file (DATABASE_TYPE=sqlite). It always lives
+# under the /root/.specflow bind mount, so it is NOT a user knob — relocate the db by pointing
+# SPECFLOW_HOME_MOUNT_PATH at a different host dir. Only host-side seeding and tests override
+# SQLITE_DB_PATH (via env) to address the same bind-mounted file by its real host path.
+CONTAINER_SQLITE_DB_PATH = "/root/.specflow/db/specflow.db"
+
 # Single source of truth for the P10Y/Compass endpoint.
 P10Y_DEFAULT_BASE_URL = "https://compass.p10y.com"
 
@@ -239,6 +245,7 @@ def _empty_str_to_none_int(cls, v: object) -> object:
     FIRESTORE_EMULATOR_HOST: Optional[str] = None  # e.g., localhost:8080 or firestore-emulator:8080
     GCP_PROJECT_ID: Optional[str] = None  # GCP project ID for Firestore
     FIRESTORE_DATABASE_NAME: str = "default"  # Firestore database name (default: "(default)")
+    SQLITE_DB_PATH: str = CONTAINER_SQLITE_DB_PATH  # container default; host seeding/tests override via env
 
     # LLM Provider Configuration
     # DEFAULT_PROVIDER is derived from the API keys below — see the computed_field.

backend/app/core/enums.py

@@ -23,3 +23,4 @@ class DatabaseType(StrEnum):
     MEMORY = "memory"
     EMULATOR = "emulator"
     FIRESTORE = "firestore"
+    SQLITE = "sqlite"

backend/app/core/local_identity.py

@@ -2,11 +2,11 @@
 Local identity constants for single-user local auth mode.
 
 Single source of truth for the sentinel api_keys document used by
-LocalAuthMiddleware (Phase 3) and init_firestore.py sentinel seeding (Phase 5).
+LocalAuthMiddleware (Phase 3) and init_db.py sentinel seeding (Phase 5).
 """
 
 LOCAL_API_KEY_DOC_ID: str = "local"
-"""Firestore document-id for the local sentinel api_keys doc."""
+"""Document-id for the local sentinel api_keys doc."""
 
 LOCAL_KEY_UID: str = "00000000-10ca-0000-0000-000000000001"
 """

backend/app/database/__init__.py

@@ -2,9 +2,10 @@
 Database abstraction layer for SpecFlow backend.
 
 Provides a unified interface for database operations with support for:
-- Production: Cloud Firestore
-- Development: Firestore Emulator
+- Local / Docker dev (default): SQLite (single-writer, persistent)
+- Production, or connecting to an already-hosted GCP instance: Firestore
 - Testing: In-memory database
+- Manual Firestore-emulator use (not started by docker-compose): Emulator
 
 The factory pattern selects the appropriate implementation based on environment configuration.
 
@@ -34,6 +35,7 @@
 from app.database.memory import InMemoryDatabase
 from app.database.firestore import FirestoreDatabase
 from app.database.emulator import EmulatorDatabase
+from app.database.sqlite import SqliteDatabase
 from app.database.factory import get_database, reset_database, clear_test_data
 from app.database.dependencies import get_db
 
@@ -47,6 +49,7 @@
     "InMemoryDatabase",
     "FirestoreDatabase",
     "EmulatorDatabase",
+    "SqliteDatabase",
     # Factory functions (recommended way to get database instance)
     "get_database",
     "reset_database",

backend/app/database/factory.py

@@ -14,6 +14,7 @@
 from app.database.memory import InMemoryDatabase
 from app.database.emulator import EmulatorDatabase
 from app.database.firestore import FirestoreDatabase
+from app.database.sqlite import SqliteDatabase
 
 
 # Singleton instance
@@ -29,17 +30,19 @@ def get_database() -> IDatabase:
 
     The implementation is selected based on the DATABASE_TYPE setting:
     - "memory": InMemoryDatabase (for unit tests)
-    - "emulator": EmulatorDatabase (for local development with Firestore Emulator)
-    - "firestore": FirestoreDatabase (for production GCP Firestore)
-
+    - "sqlite": SqliteDatabase (local/Docker-dev default; single-writer, persistent)
+    - "emulator": EmulatorDatabase (manual Firestore-emulator use; not started by docker-compose)
+    - "firestore": FirestoreDatabase (production, or connecting to an already-hosted GCP instance)
+
     Returns:
         IDatabase: Configured database instance
-        
+
     Raises:
         ValueError: If DATABASE_TYPE is invalid
-        
+
     Environment Variables:
-        DATABASE_TYPE: Type of database to use (memory|emulator|firestore)
+        DATABASE_TYPE: Type of database to use (memory|sqlite|emulator|firestore)
+        SQLITE_DB_PATH: SQLite file path (required for sqlite mode)
         FIRESTORE_EMULATOR_HOST: Emulator host:port (required for emulator mode)
         GCP_PROJECT_ID: GCP project ID (optional for firestore mode)
 
@@ -59,6 +62,8 @@ def get_database() -> IDatabase:
 
     if db_type == DatabaseType.MEMORY:
         _database_instance = InMemoryDatabase()
+    elif db_type == DatabaseType.SQLITE:
+        _database_instance = SqliteDatabase(db_path=settings.SQLITE_DB_PATH)
     elif db_type == DatabaseType.EMULATOR:
         if not settings.FIRESTORE_EMULATOR_HOST:
             raise ValueError(
@@ -79,7 +84,7 @@ def get_database() -> IDatabase:
     else:
         raise ValueError(
             f"Invalid DATABASE_TYPE: {db_type}. "
-            f"Must be one of: memory, emulator, firestore"
+            f"Must be one of: memory, sqlite, emulator, firestore"
         )
 
     return _database_instance
@@ -129,10 +134,10 @@ def clear_test_data(collections: Optional[list[str]] = None) -> None:
 
     # Check if we're in a safe environment
     db_type = settings.DATABASE_TYPE
-    if db_type not in (DatabaseType.MEMORY, DatabaseType.EMULATOR):
+    if db_type not in (DatabaseType.MEMORY, DatabaseType.EMULATOR, DatabaseType.SQLITE):
         raise RuntimeError(
-            f"clear_test_data() should only be used with 'memory' or 'emulator' databases. "
-            f"Current database type: {db_type}"
+            f"clear_test_data() should only be used with 'memory', 'emulator', or 'sqlite' "
+            f"databases. Current database type: {db_type}"
         )
 
     # Call clear_all if the database supports it