griddynamics · mkonopelski-gd · Jul 1, 2026 · Jul 1, 2026 · Jul 1, 2026 · Jul 2, 2026
@@ -48,30 +48,55 @@ AUTH_MODE=local
 
 # DATABASE_TYPE
 #   What:  Which persistence backend to use for state and session data.
-#          "emulator" = bundled Firestore emulator container (no GCP needed).
-#          "firestore" = real GCP Firestore (production).
+#          "sqlite" = local file, no GCP needed, no separate container (default).
+#          "firestore" = connect to an already-hosted, GCP-managed Firestore
+#                        instance (production, or your own hosted instance).
+#          "emulator" = connect to a manually-run Firestore emulator process
+#                       (SpecFlow does not start one for you).
 #          "memory" = in-memory only (unit tests).
-#   How:   Leave as "emulator" for quickstart. No GCP credentials required.
-DATABASE_TYPE=emulator
+#   How:   Leave as "sqlite" for quickstart. No GCP credentials required.
+DATABASE_TYPE=sqlite
+
+# SQLITE_DB_PATH
+#   What:  Container-internal path to the SQLite file (only used when
+#          DATABASE_TYPE=sqlite), nested under a db/ subdir so the specflow.db /
+#          -wal / -shm files stay separate from other files (e.g. config.json)
+#          docker-compose bind-mounts into the same ~/.specflow/ host directory
+#          (SPECFLOW_HOME_MOUNT_PATH below) — one central database shared across
+#          every local project/MCP session on this machine, the same way the
+#          Firestore emulator used to be one shared local instance. The parent
+#          dir is created automatically on first backend startup if missing.
+#   How:   Leave this commented out. It is a CONTAINER-INTERNAL path and is already
+#          defaulted by docker-compose; setting it here has no effect on host-side
+#          seeding (which derives its path from SPECFLOW_HOME_MOUNT_PATH). Only
+#          uncomment + change it if you also change the docker-compose bind-mount.
+# SQLITE_DB_PATH=/root/.specflow/db/specflow.db
+
+# SPECFLOW_HOME_MOUNT_PATH
+#   What:  Host-side directory bind-mounted into the backend container at
+#          /root/.specflow (SQLITE_DB_PATH's grandparent). Must be on real
+#          (block) storage, never NFS/Filestore — SQLite's file locking is
+#          unsafe over NFS. Docker creates this host directory automatically
+#          on first `docker compose up` if it doesn't already exist.
+#   How:   Leave unset to default to ~/.specflow.
+# SPECFLOW_HOME_MOUNT_PATH=
 
 # FIRESTORE_EMULATOR_HOST
-#   What:  Network address of the Firestore emulator. Only used when
-#          DATABASE_TYPE=emulator.
-#   How:   Leave as "localhost:8080". Host-side scripts (init_firestore.py)
-#          connect here. Inside the backend container, docker-compose
-#          automatically injects "firestore-emulator:8080".
+#   What:  Network address of a manually-run Firestore emulator. Only used
+#          when DATABASE_TYPE=emulator.
+#   How:   Leave as "localhost:8080" if you run one yourself; otherwise unused.
 FIRESTORE_EMULATOR_HOST=localhost:8080
 
 # GCP_PROJECT_ID
-#   What:  GCP project identifier passed to the Firestore client.
-#          For the emulator this is an arbitrary string (not validated).
-#          For production Firestore, use your real GCP project ID.
-#   How:   Leave as "local-dev" for quickstart.
+#   What:  GCP project identifier passed to the Firestore client. Only used
+#          when DATABASE_TYPE=firestore or emulator.
+#   How:   Leave as "local-dev" for quickstart (sqlite ignores this).
 GCP_PROJECT_ID=local-dev
 
 # FIRESTORE_DATABASE_NAME
-#   What:  Logical database name within the Firestore project.
-#   How:   Leave as "specflow" for quickstart.
+#   What:  Logical database name within the Firestore project. Only used
+#          when DATABASE_TYPE=firestore or emulator.
+#   How:   Leave as "specflow" for quickstart (sqlite ignores this).
 FIRESTORE_DATABASE_NAME=specflow
 
 # WORKSPACE_BASE_PATH / AGENT_LOGS_BASE_PATH
@@ -82,13 +107,6 @@ FIRESTORE_DATABASE_NAME=specflow
 WORKSPACE_BASE_PATH=/workspaces
 AGENT_LOGS_BASE_PATH=/agent_logs
 
-# FIRESTORE_EXPORT_INTERVAL_SECONDS
-#   What:  How often (in seconds) the emulator sidecar exports Firestore state
-#          to ./workspaces/firestore_emulator/ for crash durability. On restart
-#          the emulator restores from the last export automatically.
-#   How:   Leave as 25. Reset all exported data with: ./specflow-init.sh --reset-local-db
-FIRESTORE_EXPORT_INTERVAL_SECONDS=25
-
 # USER_EMAIL / GIT_USER_EMAIL
 #   What:  Your email identity used for MCP config, notification emails, and
 #          git commit authorship on workspaces.

@@ -3,22 +3,32 @@
 # Default target
 .DEFAULT_GOAL := build
 
-# One-command local quickstart: bootstrap emulator + backend, seed Firestore, emit MCP config
+# One-command local quickstart: bootstrap backend, seed the database, emit MCP config
 quickstart:
 	@./specflow-init.sh $(ARGS)
 
 # Allow WORKSPACE_MOUNT_PATH to be passed as a parameter
 WORKSPACE_MOUNT_PATH ?= ./workspaces
 
 BACKEND_URL ?= http://localhost:8000
+# Local/Docker default. Override to "firestore" to connect to an already-hosted, GCP-managed
+# Firestore instance, or "emulator" to connect to a manually-run Firestore emulator process —
+# SpecFlow itself never deploys/manages either locally.
+DATABASE_TYPE ?= sqlite
+# Central SQLite file, bind-mounted into the backend container at the same path (see
+# docker-compose.yml) — one database shared across every local project/MCP session, matching
+# the old shared Firestore-emulator model. Host-side scripts (init_db.py, tests) read/write
+# the exact same file directly; no docker exec needed.
+SPECFLOW_HOME_PATH ?= $(HOME)/.specflow
+SQLITE_DB_PATH ?= $(SPECFLOW_HOME_PATH)/db/specflow.db
 FIRESTORE_EMULATOR_HOST ?= localhost:8080
 # Must match docker-compose.yml defaults so host-side seeding/tests see the same
 # named Firestore database as the backend container (quickstart sets these via specflow-init.sh).
 GCP_PROJECT_ID ?= local-dev
 FIRESTORE_DATABASE_NAME ?= specflow
 E2E_WORKSPACE_COUNT ?= 3
 
-# Workspace-pool repos used to prefill the test Firestore (init_firestore.py). REQUIRED: there are
+# Workspace-pool repos used to prefill the test database (init_db.py). REQUIRED: there are
 # no default repos, and SKIP_MODE still clones each repo during allocation — so point this at a JSON
 # list of YOUR test repos. The e2e targets refuse to run without it.
 # Schema: [{"workspace_id": "ws-01-1", "repo_url": "https://github.com/org/repo",
@@ -30,33 +40,33 @@ E2E_WORKSPACE_COUNT ?= 3
 # Or point at any path explicitly:  make skip-mode-e2e-tests E2E_WORKSPACE_CONFIG=my-test-repos.json
 E2E_WORKSPACE_CONFIG ?= $(wildcard e2e-workspace-config.json)
 # --yes keeps re-runs non-interactive; --workspace-config supplies the (required) workspace pool.
-INIT_FIRESTORE_ARGS := --yes
+INIT_DB_ARGS := --yes
 ifneq ($(strip $(E2E_WORKSPACE_CONFIG)),)
-INIT_FIRESTORE_ARGS += --workspace-config $(abspath $(E2E_WORKSPACE_CONFIG))
+INIT_DB_ARGS += --workspace-config $(abspath $(E2E_WORKSPACE_CONFIG))
 endif
 
 # ── Isolated local-testing stack ───────────────────────────────────────────────────────────
 # Contributor test runs (e2e + integration) use a SEPARATE docker-compose project, container
-# names, host ports, and workspace/Firestore volume from a self-hosted quickstart deployment
-# (quickstart uses the default project + ./workspaces). So a test run never clobbers a running
-# quickstart: `make stop` only tears down the test stack and removes its ephemeral ./.specflow-test
-# state.
+# names, host ports, and workspace/database paths from a self-hosted quickstart deployment
+# (quickstart uses the default project + ./workspaces + ~/.specflow). So a test run never
+# clobbers a running quickstart OR the real central SQLite database: `make stop` tears down
+# the test stack and removes its ephemeral ./.specflow-test state (which nests its own
+# isolated specflow-home/db/specflow.db, cleaned up the same way).
 #
 # Applied as target-specific *exported* vars so they also reach prerequisite targets
 # (run-detached / run-detached-skip) and sub-makes (`$(MAKE) stop`, `$(MAKE) e2e-setup`).
 SPECFLOW_BACKEND_CONTAINER ?= specflow-backend
 TEST_WORKSPACE_MOUNT_PATH := ./.specflow-test
+TEST_SPECFLOW_HOME_PATH := $(TEST_WORKSPACE_MOUNT_PATH)/specflow-home
 TEST_STACK_TARGETS := e2e-setup skip-mode-e2e-tests contract-validation-e2e-tests shutdown-recovery-e2e-tests real-e2e-tests integration-tests stop stop-test
 $(TEST_STACK_TARGETS): export COMPOSE_PROJECT_NAME := specflow-test
 $(TEST_STACK_TARGETS): export WORKSPACE_MOUNT_PATH := $(TEST_WORKSPACE_MOUNT_PATH)
 $(TEST_STACK_TARGETS): export SPECFLOW_BACKEND_CONTAINER := specflow-test-backend
-$(TEST_STACK_TARGETS): export SPECFLOW_FIRESTORE_CONTAINER := specflow-test-firestore-emulator
-$(TEST_STACK_TARGETS): export SPECFLOW_FIRESTORE_EXPORTER_CONTAINER := specflow-test-firestore-exporter
 $(TEST_STACK_TARGETS): export SPECFLOW_MCP_CONTAINER := specflow-test-mcp-server
 $(TEST_STACK_TARGETS): export SPECFLOW_BACKEND_PORT := 18000
-$(TEST_STACK_TARGETS): export SPECFLOW_FIRESTORE_PORT := 18080
 $(TEST_STACK_TARGETS): export BACKEND_URL := http://localhost:18000
-$(TEST_STACK_TARGETS): export FIRESTORE_EMULATOR_HOST := localhost:18080
+$(TEST_STACK_TARGETS): export SPECFLOW_HOME_MOUNT_PATH := $(TEST_SPECFLOW_HOME_PATH)
+$(TEST_STACK_TARGETS): export SQLITE_DB_PATH := $(TEST_SPECFLOW_HOME_PATH)/db/specflow.db
 $(TEST_STACK_TARGETS): export GCP_PROJECT_ID := $(GCP_PROJECT_ID)
 $(TEST_STACK_TARGETS): export FIRESTORE_DATABASE_NAME := $(FIRESTORE_DATABASE_NAME)
 
@@ -87,36 +97,36 @@ build:
 	docker-compose build
 
 run:
-	@echo "🚀 Starting services in DEV mode (with Firestore Emulator)..."
+	@echo "🚀 Starting services in DEV mode (DATABASE_TYPE=$(DATABASE_TYPE))..."
 	@echo "📁 Using WORKSPACE_MOUNT_PATH: $(WORKSPACE_MOUNT_PATH)"
-	@echo "💾 Database: Firestore Emulator (no GCP credentials needed)"
+	@echo "💾 Database: SQLite at $(SQLITE_DB_PATH) (no GCP credentials needed)"
 	WORKSPACE_MOUNT_PATH=$(WORKSPACE_MOUNT_PATH) docker-compose up --no-build
 
 # Run with SKIP_MODE enabled (agents return immediately without execution)
 run-skip:
 	@echo "🚀 Starting services in DEV mode with SKIP_MODE enabled..."
 	@echo "📁 Using WORKSPACE_MOUNT_PATH: $(WORKSPACE_MOUNT_PATH)"
-	@echo "💾 Database: Firestore Emulator (no GCP credentials needed)"
+	@echo "💾 Database: SQLite at $(SQLITE_DB_PATH) (no GCP credentials needed)"
 	@echo "⏭️  Agent execution: SKIPPED (testing mode)"
 	WORKSPACE_MOUNT_PATH=$(WORKSPACE_MOUNT_PATH) SKIP_AGENT_EXECUTION=true docker-compose up --no-build
 
 # Run in detached mode
 run-detached: build
 	@echo "🚀 Starting services in background (DEV mode)..."
 	@echo "📁 Using WORKSPACE_MOUNT_PATH: $(WORKSPACE_MOUNT_PATH)"
-	@echo "💾 Database: Firestore Emulator"
+	@echo "💾 Database: SQLite at $(SQLITE_DB_PATH)"
 	WORKSPACE_MOUNT_PATH=$(WORKSPACE_MOUNT_PATH) docker-compose up -d --no-build
 
 # Run in detached mode with SKIP_MODE (same as run-detached: build then up — cache-friendly)
 run-detached-skip: build
 	@echo "🚀 Starting services in background (DEV mode) with SKIP_MODE..."
 	@echo "📁 Using WORKSPACE_MOUNT_PATH: $(WORKSPACE_MOUNT_PATH)"
-	@echo "💾 Database: Firestore Emulator"
+	@echo "💾 Database: SQLite at $(SQLITE_DB_PATH)"
 	@echo "⏭️  Agent execution: SKIPPED (testing mode)"
 	WORKSPACE_MOUNT_PATH=$(WORKSPACE_MOUNT_PATH) SKIP_AGENT_EXECUTION=true docker-compose up -d --no-build
 
 # Stop ONLY the isolated local-testing stack (project: specflow-test) and wipe its ephemeral
-# workspace/Firestore state. Quickstart is stopped outside this Make target.
+# workspace/database state. Quickstart is stopped outside this Make target.
 stop:
 	@echo "🛑 Stopping the isolated local-testing stack (project: specflow-test)..."
 	docker-compose down --timeout 90
@@ -221,19 +231,35 @@ format:
 	@cd backend && uv run ruff check . --fix
 	@echo "✅ Code formatted"
 
-# Initialize Firestore (with emulator)
+# Initialize the active database backend (sqlite by default; override DATABASE_TYPE for
+# firestore/emulator). Runs host-side against the same file the backend container has
+# bind-mounted (sqlite) or the same emulator host:port (emulator) — no docker exec needed.
+init-db:
+	$(require-e2e-workspace-config)
+	@echo "🔧 Initializing database (DATABASE_TYPE=$(DATABASE_TYPE))..."
+	@cd backend && \
+		DATABASE_TYPE=$(DATABASE_TYPE) \
+		SQLITE_DB_PATH=$(SQLITE_DB_PATH) \
+		FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) \
+		uv run scripts/init_db.py $(INIT_DB_ARGS)
+
+# Backward-compatible alias.
 init-firestore:
+	@$(MAKE) init-db
+
+# Initialize the active database backend (dry run)
+init-db-dry:
 	$(require-e2e-workspace-config)
-	@echo "🔧 Initializing Firestore database..."
-	@echo "⚠️  Using FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST)"
-	@cd backend && FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) uv run scripts/init_firestore.py $(INIT_FIRESTORE_ARGS)
+	@echo "🔧 Dry run: Initializing database (DATABASE_TYPE=$(DATABASE_TYPE))..."
+	@cd backend && \
+		DATABASE_TYPE=$(DATABASE_TYPE) \
+		SQLITE_DB_PATH=$(SQLITE_DB_PATH) \
+		FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) \
+		uv run scripts/init_db.py --dry-run $(INIT_DB_ARGS)
 
-# Initialize Firestore (dry run)
+# Backward-compatible alias.
 init-firestore-dry:
-	$(require-e2e-workspace-config)
-	@echo "🔧 Dry run: Initializing Firestore database..."
-	@echo "⚠️  Using FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST)"
-	@cd backend && FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) uv run scripts/init_firestore.py --dry-run $(INIT_FIRESTORE_ARGS)
+	@$(MAKE) init-db-dry
 
 # Create estimation repositories
 # Usage: make create-repos START=7 END=9
@@ -268,15 +294,17 @@ unit-tests:
 	@cd mcp_server && uv run pytest tests/ -v
 	@echo "✅ Unit tests passed"
 
-# Run integration tests (with Firestore Emulator)
+# Run integration tests (sqlite by default; override DATABASE_TYPE=emulator/firestore)
 integration-tests:
 	@$(MAKE) stop
 	@$(MAKE) run-detached
 	@echo "⏳ Waiting for services to be ready..."
 	@sleep 5
-	@echo "🧪 Running integration tests (Firestore Emulator, database=$(FIRESTORE_DATABASE_NAME))..."
+	@echo "🧪 Running integration tests (DATABASE_TYPE=$(DATABASE_TYPE))..."
 	@cd backend && \
-		DATABASE_TYPE=emulator \
+		DATABASE_TYPE=$(DATABASE_TYPE) \
+		SQLITE_DB_PATH=$(SQLITE_DB_PATH) \
+		FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) \
 		AUTH_MODE=api_key \
 		RUN_GIT_INTEGRATION_TESTS=1 \
 		uv run pytest test/ -v --cov=app
@@ -302,10 +330,13 @@ e2e-setup:
 		echo "   Attempt $$i/10..."; \
 		sleep 2; \
 	done
-	@echo "🔧 Initializing Firestore database (database=$(FIRESTORE_DATABASE_NAME))..."
+	@echo "🔧 Initializing database (DATABASE_TYPE=$(DATABASE_TYPE))..."
 	@cd backend && \
 		GITHUB_TOKEN=$${GITHUB_TOKEN:-} \
-		uv run scripts/init_firestore.py $(INIT_FIRESTORE_ARGS) || (echo "⚠️  Firestore initialization failed. Services may still be starting. Retry with: make init-firestore" && exit 1)
+		DATABASE_TYPE=$(DATABASE_TYPE) \
+		SQLITE_DB_PATH=$(SQLITE_DB_PATH) \
+		FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) \
+		uv run scripts/init_db.py $(INIT_DB_ARGS) || (echo "⚠️  Database initialization failed. Services may still be starting. Retry with: make init-db" && exit 1)
 	@echo "🔑 Fetching API key..."
 	@cd backend && \
 		uv run python ../scripts/get-api-key.py || (echo "⚠️  Could not fetch API key" && exit 1)
@@ -317,8 +348,8 @@ e2e-setup:
 	@echo "=========================================="
 	@echo ""
 	@echo "📋 Services running:"
-	@echo "  - Backend API:     $(BACKEND_URL)"
-	@echo "  - Firestore Emulator: $(FIRESTORE_EMULATOR_HOST)"
+	@echo "  - Backend API: $(BACKEND_URL)"
+	@echo "  - Database:    $(DATABASE_TYPE) ($(SQLITE_DB_PATH))"
 	@echo ""
 	@echo "📁 Example specifications created at:"
 	@echo "  /tmp/specflow-e2e-specs"
@@ -410,7 +441,7 @@ help:
 	@echo "  make quickstart ARGS='--skip-repos'             - Skip GitHub repo creation (supply .specflow-local/workspaces.json)"
 	@echo "  make build                                      - Build base image and all services (default)"
 	@echo "  make base                                       - Build only the base image"
-	@echo "  make run                                        - Start in DEV mode (Firestore Emulator)"
+	@echo "  make run                                        - Start in DEV mode (SQLite, no GCP credentials needed)"
 	@echo "  make run-skip                                   - Start in DEV mode with SKIP_MODE (agents return immediately)"
 	@echo "  make run WORKSPACE_MOUNT_PATH=/path             - Start with custom workspace mount"
 	@echo "  make run-detached                               - Start in background (DEV mode)"
@@ -432,16 +463,16 @@ help:
 	@echo "  make secret-scan-history                        - Run secret scans including full local git history"
 	@echo "  make format                                     - Format code with ruff"
 	@echo "  make unit-tests                                 - Run unit tests (in-memory database, fast)"
-	@echo "  make integration-tests                          - Run integration tests (Firestore Emulator)"
+	@echo "  make integration-tests                          - Run integration tests (SQLite by default; DATABASE_TYPE=emulator|firestore to override)"
 	@echo "  (e2e + integration tests run in an isolated ephemeral stack: project specflow-test, mount ./.specflow-test)"
-	@echo "  make e2e-setup                                  - Setup E2E environment (starts services, initializes Firestore, creates example specs)"
+	@echo "  make e2e-setup                                  - Setup E2E environment (starts services, initializes the database, creates example specs)"
 	@echo "  make skip-mode-e2e-tests                        - Fast E2E of the MCP tool sequence + contract gate (SKIP mode)"
 	@echo "      E2E_WORKSPACE_CONFIG=path.json              - REQUIRED: prefill the test pool with your own repos (no defaults)"
 	@echo "  make contract-validation-e2e-tests              - E2E: contract rejections reject before allocating (no orphan workspaces)"
 	@echo "  make shutdown-recovery-e2e-tests                - E2E: restart backend mid-run, verify graceful shutdown + boot recovery"
 	@echo "  make real-e2e-tests                             - Full real-agent E2E (slow, 30-90 min)"
-	@echo "  make init-firestore-dry                         - Initialize Firestore (dry run, shows what would be done)"
-	@echo "  make init-firestore                             - Initialize Firestore database (requires FIRESTORE_EMULATOR_HOST)"
+	@echo "  make init-db-dry                                - Initialize the active database (dry run, shows what would be done)"
+	@echo "  make init-db                                    - Initialize the active database (SQLite by default; DATABASE_TYPE to override)"
 	@echo "  make create-repos START=7 END=9                 - Create generation workspace repositories"
 	@echo "  make create-repos START=1 END=3 PREFIX=test     - Create repos with custom prefix"
 	@echo ""
@@ -456,7 +487,9 @@ help:
 	@echo "  make ops-retry-run generation_id=X BACKEND_URL=http://host:8000  - Override backend URL"
 	@echo ""
 	@echo "Database Modes:"
-	@echo "  DEV mode:    Uses Firestore Emulator (no GCP credentials needed)"
+	@echo "  DEV mode:    Uses SQLite (no GCP credentials needed, single central db at ~/.specflow/db/specflow.db)"
+	@echo "  Override:    DATABASE_TYPE=firestore to connect to an already-hosted GCP Firestore instance"
+	@echo "               DATABASE_TYPE=emulator to connect to a manually-run Firestore emulator process"
 	@echo ""
 	@echo "SKIP_MODE:"
 	@echo "  When enabled, agent_query returns immediately with 'SKIP_MODE' response"

@@ -148,7 +148,7 @@ Stop the local stack:
 docker compose down --timeout 90
 ```
 
-Reset local Firestore state and reseed:
+Reset the local SQLite database and reseed:
 
 ```bash
 specflow init --reset-local-db