diff --git a/README.md b/README.md index d50ef917..ac168376 100644 --- a/README.md +++ b/README.md @@ -43,14 +43,20 @@ Use the canonical setup guide: Fast path: ```sh -cp elf.example.toml elf.toml -psql "" -f sql/init.sql -./qdrant/init.sh -cargo run -p elf-worker -- -c elf.toml -cargo run -p elf-api -- -c elf.toml -cargo run -p elf-mcp -- -c elf.toml +docker compose -f docker-compose.yml up -d postgres qdrant + +# Terminal 1 +cargo run -p elf-api -- -c config/local/elf.docker.toml + +# Terminal 2 +cargo run -p elf-worker -- -c config/local/elf.docker.toml + +# Terminal 3 +curl -fsS http://127.0.0.1:51892/health ``` +For provider-backed development, copy `elf.example.toml` to `elf.toml` and fill the provider blocks. + ## Architecture ```mermaid diff --git a/config/local/elf.docker.toml b/config/local/elf.docker.toml new file mode 100644 index 00000000..ec186717 --- /dev/null +++ b/config/local/elf.docker.toml @@ -0,0 +1,213 @@ +[service] +admin_bind = "127.0.0.1:51891" +http_bind = "127.0.0.1:51892" +log_level = "info" +mcp_bind = "127.0.0.1:51893" + +[storage.postgres] +dsn = "postgres://elf_dev:elf_dev_password@127.0.0.1:51888/elf_local" +pool_max_conns = 10 + +[storage.qdrant] +collection = "elf_local_notes" +docs_collection = "elf_local_doc_chunks" +url = "http://127.0.0.1:51890" +vector_dim = 256 + +[mcp] +agent_id = "local-agent" +project_id = "local-project" +read_profile = "private_plus_project" +tenant_id = "local-tenant" + +[providers.embedding] +api_base = "http://127.0.0.1" +api_key = "local-dev-placeholder" +default_headers = {} +dimensions = 256 +model = "local-hash" +path = "/embeddings" +provider_id = "local" +timeout_ms = 1_000 + +[providers.rerank] +api_base = "http://127.0.0.1" +api_key = "local-dev-placeholder" +default_headers = {} +model = "local-token-overlap" +path = "/rerank" +provider_id = "local" +timeout_ms = 1_000 + +[providers.llm_extractor] +api_base = "http://127.0.0.1" +api_key = "local-dev-placeholder" +default_headers = {} +model = "local-disabled" +path = "/chat/completions" +provider_id = "local-disabled" +temperature = 0.0 +timeout_ms = 1_000 + +[scopes] +allowed = ["agent_private", "org_shared", "project_shared"] + +[scopes.read_profiles] +all_scopes = ["agent_private", "org_shared", "project_shared"] +private_only = ["agent_private"] +private_plus_project = ["agent_private", "project_shared"] + +[scopes.precedence] +agent_private = 30 +org_shared = 10 +project_shared = 20 + +[scopes.write_allowed] +agent_private = true +org_shared = true +project_shared = true + +[memory] +candidate_k = 60 +dup_sim_threshold = 0.92 +max_note_chars = 240 +max_notes_per_add_event = 3 +top_k = 12 +update_sim_threshold = 0.85 + +[memory.policy] + +[[memory.policy.rules]] +min_confidence = 0.9 +min_importance = 0.75 +note_type = "preference" +scope = "agent_private" + +[chunking] +enabled = true +max_tokens = 512 +overlap_tokens = 128 +tokenizer_repo = "config/local/tokenizer.wordlevel.json" + +[search.expansion] +include_original = true +max_queries = 4 +mode = "off" + +[search.dynamic] +min_candidates = 10 +min_top_score = 0.12 + +[search.prefilter] +max_candidates = 0 + +[search.cache] +enabled = false +expansion_ttl_days = 7 +max_payload_bytes = 262_144 +rerank_ttl_days = 7 + +[search.explain] +candidate_retention_days = 2 +capture_candidates = false +retention_days = 7 +write_mode = "outbox" + +[search.recursive] +enabled = false +max_children_per_node = 4 +max_depth = 2 +max_nodes_per_scope = 32 +max_total_nodes = 256 + +[search.graph_context] +enabled = false +max_evidence_notes_per_fact = 16 +max_facts_per_item = 16 + +[ranking] +recency_tau_days = 60.0 +tie_breaker_weight = 0.1 + +[ranking.deterministic] +enabled = false + +[ranking.deterministic.lexical] +enabled = false +max_query_terms = 16 +max_text_terms = 1_024 +min_ratio = 0.3 +weight = 0.05 + +[ranking.deterministic.hits] +enabled = false +half_saturation = 8.0 +last_hit_tau_days = 14.0 +weight = 0.05 + +[ranking.deterministic.decay] +enabled = false +tau_days = 30.0 +weight = 0.05 + +[ranking.blend] +enabled = true +rerank_normalization = "rank" +retrieval_normalization = "rank" + +[[ranking.blend.segments]] +max_retrieval_rank = 3 +retrieval_weight = 0.8 + +[[ranking.blend.segments]] +max_retrieval_rank = 10 +retrieval_weight = 0.5 + +[[ranking.blend.segments]] +max_retrieval_rank = 1_000_000 +retrieval_weight = 0.2 + +[ranking.diversity] +enabled = true +max_skips = 64 +mmr_lambda = 0.7 +sim_threshold = 0.88 + +[ranking.retrieval_sources] +fusion_priority = 1 +fusion_weight = 1.0 +structured_field_priority = 0 +structured_field_weight = 1.0 + +[lifecycle.ttl_days] +constraint = 0 +decision = 0 +fact = 180 +plan = 14 +preference = 0 +profile = 0 + +[lifecycle] +purge_deleted_after_days = 30 +purge_deprecated_after_days = 180 + +[security] +auth_keys = [] +auth_mode = "off" +bind_localhost_only = true +evidence_max_quote_chars = 320 +evidence_max_quotes = 2 +evidence_min_quotes = 1 +redact_secrets_on_write = true +reject_non_english = true + +[context] +scope_boost_weight = 0.0 + +[context.project_descriptions] +"local-tenant:local-project" = "Local ELF development stack." + +[context.scope_descriptions] +agent_private = "Local private notes for one development agent." +org_shared = "Local organization-shared development notes." +project_shared = "Local project-shared development notes." diff --git a/config/local/tokenizer.wordlevel.json b/config/local/tokenizer.wordlevel.json new file mode 100644 index 00000000..631ac318 --- /dev/null +++ b/config/local/tokenizer.wordlevel.json @@ -0,0 +1,19 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": { + "type": "Whitespace" + }, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": { + "[UNK]": 0 + }, + "unk_token": "[UNK]" + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..69914abb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +name: elf-local-dev + +services: + postgres: + image: pgvector/pgvector:pg18 + environment: + POSTGRES_DB: elf_local + POSTGRES_USER: elf_dev + POSTGRES_PASSWORD: elf_dev_password + ports: + - "127.0.0.1:51888:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U elf_dev -d elf_local"] + interval: 10s + timeout: 5s + retries: 10 + volumes: + - elf-postgres-data:/var/lib/postgresql/data + + qdrant: + image: qdrant/qdrant:v1.16.3 + ports: + - "127.0.0.1:51889:6333" + - "127.0.0.1:51890:6334" + volumes: + - elf-qdrant-data:/qdrant/storage + +volumes: + elf-postgres-data: + elf-qdrant-data: diff --git a/docs/guide/agent-setup.md b/docs/guide/agent-setup.md index fa166acd..e4e81473 100644 --- a/docs/guide/agent-setup.md +++ b/docs/guide/agent-setup.md @@ -2,8 +2,8 @@ Goal: Help an agent install and run ELF locally with minimal back-and-forth. Read this when: You need a practical local setup flow from an existing repository checkout. -Inputs: This repository checkout plus access to local Postgres, Qdrant, and provider credentials. -Depends on: `Makefile.toml`, `elf.example.toml`, and `docs/guide/getting_started.md`. +Inputs: This repository checkout plus Docker Compose or separately managed Postgres/Qdrant, and optional provider credentials. +Depends on: `Makefile.toml`, `docker-compose.yml`, `config/local/elf.docker.toml`, `elf.example.toml`, and `docs/guide/getting_started.md`. Verification: ELF services start, required dependencies are reachable, and the local workflow can continue. This guide is written for AI agents helping a human operator install and run ELF locally with minimal back-and-forth. @@ -25,9 +25,12 @@ ELF requires: Important: The ELF config has no implicit defaults. All required config fields must be explicitly present in your TOML. -## Minimal Owner Inputs (Ask These) +## Minimal Owner Inputs -Ask the owner for: +For the checked-in Docker local stack, no owner inputs are required. Use `docker-compose.yml` +and `config/local/elf.docker.toml` from `docs/guide/getting_started.md`. + +For separately managed dependencies or provider-backed development, ask the owner for: 1. Postgres DSN for the target database (for example `postgres://user:pass@host:5432/elf`). 2. Qdrant endpoints: @@ -51,9 +54,10 @@ Then set `search.expansion.mode = "off"` to avoid LLM-backed query expansion. Th The machine must have: - Rust toolchain (pinned by `rust-toolchain.toml`). +- Docker Compose for the checked-in local dependency stack, or separately running Postgres and Qdrant. - `psql` available on PATH. -- Running Postgres instance with `pgvector` installed/enabled. -- Running Qdrant instance. +- Running Postgres instance with `pgvector` installed/enabled when not using Compose. +- Running Qdrant instance when not using Compose. For the repository harness scripts: @@ -63,13 +67,19 @@ For the repository harness scripts: ## Create The Config -1. Copy the template: +For the checked-in Docker local stack, use the strict-valid local config directly: + +```sh +config/local/elf.docker.toml +``` + +For provider-backed development, copy the template: ```sh cp elf.example.toml elf.toml ``` -2. Edit `elf.toml`: +Then edit `elf.toml`: - Set `[storage.postgres].dsn` to your Postgres DSN. - Set `[storage.qdrant].url` to your Qdrant gRPC base URL. @@ -82,17 +92,20 @@ cp elf.example.toml elf.toml ## Initialize Storage -1. Initialize Postgres schema: +For the checked-in Docker local stack, start dependencies and then start `elf-api` or +`elf-worker`; the services auto-create the Postgres schema and Qdrant collections. ```sh -psql "" -f sql/init.sql +docker compose -f docker-compose.yml up -d postgres qdrant ``` -2. Initialize the Qdrant collection (REST): +When using separately managed Qdrant and you need to pre-create collections before +service startup, initialize them through the REST endpoint: ```sh export ELF_QDRANT_HTTP_URL="http://127.0.0.1:6333" export ELF_QDRANT_COLLECTION="mem_notes_v2" +export ELF_QDRANT_DOCS_COLLECTION="doc_chunks_v1" export ELF_QDRANT_VECTOR_DIM="4096" ./qdrant/init.sh ``` @@ -108,16 +121,18 @@ Notes: Start each in a separate terminal: ```sh -cargo run -p elf-worker -- -c elf.toml -cargo run -p elf-api -- -c elf.toml +cargo run -p elf-worker -- -c config/local/elf.docker.toml +cargo run -p elf-api -- -c config/local/elf.docker.toml ``` Optional: ```sh -cargo run -p elf-mcp -- -c elf.toml +cargo run -p elf-mcp -- -c config/local/elf.docker.toml ``` +Replace `config/local/elf.docker.toml` with `elf.toml` when using a provider-backed config. + ## Verify ```sh @@ -137,7 +152,7 @@ The context misranking harness creates and drops a dedicated database and Qdrant Example: ```sh -ELF_PG_DSN="postgres://postgres:postgres@127.0.0.1:51888/postgres" \ +ELF_PG_DSN="postgres://elf_dev:elf_dev_password@127.0.0.1:51888/postgres" \ ELF_QDRANT_GRPC_URL="http://127.0.0.1:51890" \ ELF_QDRANT_HTTP_URL="http://127.0.0.1:51889" \ cargo make e2e diff --git a/docs/guide/getting_started.md b/docs/guide/getting_started.md index 218fffcb..320fe95e 100644 --- a/docs/guide/getting_started.md +++ b/docs/guide/getting_started.md @@ -2,19 +2,43 @@ Goal: Provide the canonical setup and local run flow for ELF. Read this when: You are bootstrapping a local ELF environment or resetting a broken one. -Inputs: This repository checkout, provider credentials, and local Postgres/Qdrant access. -Depends on: `Makefile.toml`, `elf.example.toml`, and the relevant service binaries. +Inputs: This repository checkout, Docker Compose for local dependencies, and optional provider credentials. +Depends on: `Makefile.toml`, `docker-compose.yml`, `config/local/elf.docker.toml`, `elf.example.toml`, and the relevant service binaries. Verification: Configuration is in place and the local ELF stack can start successfully. ## Prerequisites -- Postgres with `pgvector`. -- Qdrant (REST + gRPC endpoints). -- Provider endpoints for embeddings, rerank, and extraction. +- Docker Compose for the local dependency stack, or separately managed Postgres with `pgvector` and Qdrant. +- Rust toolchain from `rust-toolchain.toml`. +- Provider endpoints only when you are testing provider-backed embeddings, rerank, query expansion, or `add_event`. -## 1. Prepare config +## 1. Start local dependencies -Copy `elf.example.toml` to `elf.toml`, then set provider and storage values. +Validate and start the local Postgres and Qdrant services. +The checked-in Compose file is local-development-only: + +- Postgres: `127.0.0.1:51888`, database `elf_local`, user `elf_dev`, password `elf_dev_password`. +- Qdrant REST: `127.0.0.1:51889`. +- Qdrant gRPC: `127.0.0.1:51890`. +- Data lives in Docker volumes `elf-postgres-data` and `elf-qdrant-data`. + +```sh +docker compose -f docker-compose.yml config >/dev/null +docker compose -f docker-compose.yml up -d postgres qdrant +docker compose -f docker-compose.yml ps +``` + +## 2. Choose config + +For local dependency smoke tests, use the checked-in Docker config directly: + +```sh +config/local/elf.docker.toml +``` + +This config is strict-valid, binds only to loopback, uses the local deterministic embedding and rerank providers, disables LLM query expansion, and contains only placeholder provider keys. Do not use `add_event` with this config until you replace `[providers.llm_extractor]` with a real local or external extractor. + +For provider-backed development, copy `elf.example.toml` to `elf.toml`, then set provider and storage values. ```sh cp elf.example.toml elf.toml @@ -24,35 +48,27 @@ Reference: - Full configuration contract: `docs/spec/system_elf_memory_service_v2.md`. -## 2. Initialize storage +## 3. Start services -Initialize Postgres schema and Qdrant collections once. -Both services now auto-create the memory/docs collections (dense+bm25 vectors) and the docs payload indexes used for filtering (`scope`, `status`, `doc_type`, `agent_id`, `updated_at`, `doc_ts`, `thread_id`, `domain`, `repo`) during startup. +Run each service in its own terminal from the repository root. +`elf-api` and `elf-worker` auto-create the Postgres schema, the Qdrant memory/docs collections, and docs payload indexes during startup. ```sh -psql "" -f sql/init.sql - -# Qdrant REST endpoint (default: 6333). In this repository's local setup, it is often mapped to 51889. -# ELF uses the gRPC endpoint at runtime (default: 6334, often mapped to 51890). -export ELF_QDRANT_HTTP_URL="http://127.0.0.1:51889" -export ELF_QDRANT_COLLECTION="mem_notes_v2" -export ELF_QDRANT_DOCS_COLLECTION="doc_chunks_v1" -export ELF_QDRANT_VECTOR_DIM="4096" -./qdrant/init.sh +cargo run -p elf-api -- -c config/local/elf.docker.toml ``` -You can still run the script manually when bootstrapping a fresh Qdrant instance, but startup is not blocked if you rely on auto-ensure. - -## 3. Start services +```sh +cargo run -p elf-worker -- -c config/local/elf.docker.toml +``` -Run each service in its own terminal. +Optional MCP server: ```sh -cargo run -p elf-worker -- -c elf.toml -cargo run -p elf-api -- -c elf.toml -cargo run -p elf-mcp -- -c elf.toml +cargo run -p elf-mcp -- -c config/local/elf.docker.toml ``` +If you are using `elf.toml` instead, replace `config/local/elf.docker.toml` with `elf.toml`. + ## 4. Inspect API contract After `elf-api` starts, the API process serves: @@ -60,7 +76,7 @@ After `elf-api` starts, the API process serves: - `GET /openapi.json` for the generated OpenAPI contract. - `GET /docs` for the Scalar API reference UI. -Use the host and port from `service.http_bind` in `elf.toml`. +Use the host and port from `service.http_bind` in your config. For example: ```sh @@ -68,7 +84,37 @@ curl -fsS http://127.0.0.1:51892/openapi.json open http://127.0.0.1:51892/docs ``` -## 5. Run retrieval evaluation +## 5. Smoke the local stack + +```sh +curl -fsS http://127.0.0.1:51892/health +``` + +Run a deterministic `add_note` smoke that does not call any LLM provider: + +```sh +curl -fsS -X POST http://127.0.0.1:51892/v2/notes/ingest \ + -H 'content-type: application/json' \ + -H 'X-ELF-Tenant-Id: local-tenant' \ + -H 'X-ELF-Project-Id: local-project' \ + -H 'X-ELF-Agent-Id: local-agent' \ + -d '{ + "scope": "agent_private", + "notes": [ + { + "type": "fact", + "key": "local_compose_stack", + "text": "The local ELF development stack runs Postgres with pgvector and Qdrant through Docker Compose.", + "importance": 0.7, + "confidence": 0.9, + "ttl_days": 14, + "source_ref": {"schema": "local_smoke/v1", "ref": {"command": "docs/guide/getting_started.md"}} + } + ] + }' +``` + +## 6. Run retrieval evaluation Use `elf-eval` with your dataset. @@ -78,7 +124,19 @@ cargo run -p elf-eval -- -c elf.toml -i path/to/eval.json For dataset format and metric details, see `docs/guide/evaluation.md`. -## 6. Development workflow +## 7. Run local checks + +With the Compose dependencies running, the context misranking harness can use the same local dependency ports: + +```sh +ELF_PG_DSN="postgres://elf_dev:elf_dev_password@127.0.0.1:51888/postgres" \ +ELF_QDRANT_GRPC_URL="http://127.0.0.1:51890" \ +ELF_QDRANT_HTTP_URL="http://127.0.0.1:51889" \ +ELF_HARNESS_VECTOR_DIM=256 \ +cargo make e2e +``` + +## 8. Development workflow Use `cargo make` tasks from repository root. @@ -96,6 +154,8 @@ Notes: Set `ELF_PG_DSN` and `ELF_QDRANT_GRPC_URL`. - `cargo make e2e` runs the context misranking harness. Set `ELF_PG_DSN`, `ELF_QDRANT_GRPC_URL`, and `ELF_QDRANT_HTTP_URL`. +- Stop local dependencies with `docker compose -f docker-compose.yml down`. + Add `-v` only when you intentionally want to delete the local development volumes. ## Related guides diff --git a/packages/elf-chunking/src/lib.rs b/packages/elf-chunking/src/lib.rs index bc0fe4a8..f1209da2 100644 --- a/packages/elf-chunking/src/lib.rs +++ b/packages/elf-chunking/src/lib.rs @@ -128,6 +128,19 @@ fn overlap_tail(text: &str, overlap_tokens: u32, tokenizer: &Tokenizer) -> Strin mod tests { use crate::ChunkingConfig; + #[test] + fn loads_local_dev_tokenizer_fixture() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../config/local/tokenizer.wordlevel.json"); + let tokenizer = crate::load_tokenizer(path.to_str().expect("Path must be valid UTF-8")) + .expect("Local dev tokenizer must load."); + let cfg = ChunkingConfig { max_tokens: 10, overlap_tokens: 2 }; + let chunks = crate::split_text("One local note. Another local note.", &cfg, &tokenizer); + + assert!(!chunks.is_empty()); + assert!(chunks[0].text.contains("local note")); + } + #[test] fn splits_into_chunks_with_overlap() { let cfg = ChunkingConfig { max_tokens: 10, overlap_tokens: 2 }; diff --git a/packages/elf-config/tests/config_validation.rs b/packages/elf-config/tests/config_validation.rs index 100a3355..26554a07 100644 --- a/packages/elf-config/tests/config_validation.rs +++ b/packages/elf-config/tests/config_validation.rs @@ -161,6 +161,23 @@ fn required_config_fields_must_be_explicit() { } } +#[test] +fn docker_local_config_is_strict_valid() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../config/local/elf.docker.toml"); + let cfg = elf_config::load(path.as_path()).expect("Docker local config must load."); + + assert_eq!( + cfg.storage.postgres.dsn, + "postgres://elf_dev:elf_dev_password@127.0.0.1:51888/elf_local" + ); + assert_eq!(cfg.storage.qdrant.url, "http://127.0.0.1:51890"); + assert_eq!(cfg.storage.qdrant.collection, "elf_local_notes"); + assert_eq!(cfg.storage.qdrant.docs_collection, "elf_local_doc_chunks"); + assert_eq!(cfg.providers.embedding.provider_id, "local"); + assert_eq!(cfg.providers.rerank.provider_id, "local"); + assert_eq!(cfg.search.expansion.mode, "off"); +} + #[test] fn reject_non_english_must_be_true() { let payload = sample_toml(false);