diff --git a/.env.example b/.env.example
index 89d72f4..164bc5b 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,10 @@
-# LLM backend: "groq" (cloud, default) or "ollama" (fully local)
+# LLM backend: "claude" (Anthropic), "groq" (cloud, compose default) or "ollama" (fully local)
LLM_BACKEND=groq
+# Anthropic — required when LLM_BACKEND=claude
+# ANTHROPIC_API_KEY=
+# ANTHROPIC_MODEL=claude-sonnet-4-6
+
# Groq API key — required when LLM_BACKEND=groq. Get one at https://console.groq.com
GROQ_API_KEY=your_groq_api_key_here
@@ -10,10 +14,21 @@ GROQ_API_KEY=your_groq_api_key_here
# OLLAMA_MODEL=qwen3.5:4b
# EMBEDDING_MODEL=nomic-embed-text
+# JWT signing secret — REQUIRED. The API refuses to start without it.
+# Generate one with: openssl rand -hex 32
+JWT_SECRET=
+
+# Encryption key for stored job-site passwords — REQUIRED. Changing it makes saved passwords
+# unreadable (users just re-enter them). Generate one with:
+# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
+CREDENTIALS_KEY=
+
+# Brave Search API — optional, enables the Level 3 web-search job source. Free tier at https://brave.com/search/api
+BRAVE_SEARCH_API_KEY=
+
# Adzuna Jobs API — optional, enables the Adzuna job source. Free tier at https://developer.adzuna.com
ADZUNA_APP_ID=your_adzuna_app_id
ADZUNA_API_KEY=your_adzuna_api_key
-# Job site login credentials for autonomous application (can also be set per-persona)
-JOB_EMAIL=your_email@example.com
-JOB_PASSWORD=your_password
+# Job-site login credentials are set per user on the Profile page (stored encrypted with
+# CREDENTIALS_KEY) — they are not configured here.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..d34382f
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,61 @@
+name: CI
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+
+concurrency:
+ group: ci-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ api:
+ name: API tests
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: api
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.13"
+ cache: pip
+ cache-dependency-path: api/requirements*.txt
+ - run: pip install -r requirements.txt -r requirements-dev.txt
+ - run: pytest
+
+ browser:
+ name: Browser-agent tests (headless Chromium)
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: api
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.13"
+ cache: pip
+ cache-dependency-path: api/requirements*.txt
+ - run: pip install -r requirements.txt -r requirements-dev.txt
+ - run: python -m playwright install --with-deps chromium
+ - run: pytest -m browser
+
+ frontend:
+ name: Frontend type-check + build
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: frontend
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: npm
+ cache-dependency-path: frontend/package-lock.json
+ - run: npm ci
+ - run: npx tsc --noEmit
+ - run: npm run build
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 05dc81c..9418f49 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -10,8 +10,9 @@
-
-
+
+
+
@@ -91,7 +92,7 @@
"RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
"RunOnceActivity.git.unshallow": "true",
"copilot.chat.legacySessionMetadata.migrated": "true",
- "git-widget-placeholder": "feature/data-retrieval",
+ "git-widget-placeholder": "feature/agent-model",
"last_opened_file_path": "C:/Users/tobio/PycharmProjects/Windrush",
"node.js.detected.package.eslint": "true",
"node.js.detected.package.tslint": "true",
diff --git a/README.md b/README.md
index 4b4366b..2795379 100644
--- a/README.md
+++ b/README.md
@@ -1,58 +1,57 @@
# Windrush
-AI career advisor that analyses CVs, scores AI automation-exposure risk, finds matching jobs, generates tailored CVs and cover letters, and autonomously submits applications via browser automation.
+AI career advisor that analyses CVs, scores AI automation-exposure risk, finds matching jobs, generates tailored CVs and cover letters, and fills in job applications with a browser agent — which you review before anything is sent.
-Built with open/cloud LLMs served through an OpenAI-compatible client (Groq or local Ollama), [browser-use](https://github.com/browser-use/browser-use), Next.js, and FastAPI.
+Built with FastAPI, Next.js, [browser-use](https://github.com/browser-use/browser-use), and a choice of LLM backend: Anthropic Claude, Groq, or a local Ollama model.
---
## What it does
-1. **Accounts & Persona** — sign up, then build a rich persona (contact details, preferences, skills, work history, education, projects, certifications, screening answers, behavioural stories).
-2. **Upload a CV (PDF)** — parsed into structured data and merged into your persona.
+1. **Accounts & Persona** — sign up, then build a rich persona (contact details, preferences, skills, work history, education, projects, certifications, screening answers, behavioural stories, custom directives).
+2. **Upload a CV (PDF)** — parsed into structured data and merged into your persona; the PDF is kept so the browser agent can attach it.
3. **AI Exposure Risk** — scores each skill and job title against the Anthropic Economic Index / O\*NET task-penetration data (~18 000 occupational tasks), with LLM-generated reasoning.
-4. **Job Feed** — discovers jobs across multiple sources, caches them locally, and ranks them by dense-vector semantic similarity to your persona and search intent.
-5. **Per-job analysis** — on demand, scores fit (LLM semantic match) and AI risk for a single role.
-6. **Tailored CV + Cover Letter** — generates structured, ATS-aware documents with a live in-app preview and PDF export (HTML/CSS → WeasyPrint).
-7. **Browser Application** — autonomously fills and submits the application form with a headless Chrome agent; falls back to interactive user control if the agent gets stuck, with a review step before submit.
-8. **Skill Roadmap** — generates 6 AI-resilient skill-development recommendations tailored to your industry and target role.
-9. **Application Tracker** — persists every application through a status lifecycle (Saved → Pending Review → Evaluated → Applied → Responded → Interview → Offer → Rejected/Discarded).
+4. **Job Feed** — searches a local job store with real filters (search terms, role/domain tags, level, location, remote) and ranks results by semantic similarity to your persona. When results are thin, live discovery across job boards runs in the background and the feed fills in as it finishes.
+5. **Paste any job link** — scrape an arbitrary listing into the same analysis flow.
+6. **Per-job analysis** — fit score (LLM semantic match) and AI risk for a single role.
+7. **Tailored CV + Cover Letter** — structured, ATS-aware documents with a live in-app preview and PDF export (HTML/CSS → WeasyPrint).
+8. **Browser Application** — a headless Chrome agent fills the application form and **stops before the final submit**. You review it in a live view (and can take over with clicks/typing), then `submit`, `done` (you submitted it yourself) or `skip`.
+9. **Skill Roadmap** — 6 AI-resilient skill-development recommendations for your industry and target role.
+10. **Application Tracker** — every application through a status lifecycle (Saved → Pending Review → Evaluated → Applied → Responded → Interview → Offer → Rejected / Discarded).
---
## Architecture
-Six Docker services:
+Four Docker services:
| Service | Stack | Port | Role |
|---|---|---|---|
-| `nginx` | nginx:alpine | 80 | Reverse proxy; disables SSE buffering |
+| `nginx` | nginx:alpine | 80 | Reverse proxy; disables buffering for SSE routes |
| `frontend` | Next.js 14, React 18, TypeScript, Tailwind | 3000 | UI |
-| `api` | Python 3.13, FastAPI, OpenAI-compat client, Playwright, WeasyPrint | 8000 | Agent pipeline + browser automation + documents |
-| `jobs-mcp` | Python, FastMCP | 8001 | MCP server wrapping the Adzuna Jobs API |
-| `civic-guardrails` | Node.js, `@civic/passthrough-mcp-server` | 8002 | MCP proxy: rate-limiting, injection detection, PII scrubbing, audit logging |
-| `ollama` | ollama/ollama | 11434 | Local LLM inference + embeddings (`OLLAMA_KEEP_ALIVE=24h`) |
+| `api` | Python 3.13, FastAPI, browser-use, WeasyPrint | 8000 | LLM features, job search, documents, browser automation |
+| `ollama` | ollama/ollama | 11434 | Embeddings for the job feed (+ optional local LLM) |
-Only `nginx` is published to the host (port 80); every other service is reached through it. The `ollama` service has an NVIDIA GPU reservation enabled in `docker-compose.yml` — remove it if you don't have a GPU.
+Only `nginx` is published to the host (port 80). The `ollama` service has an NVIDIA GPU reservation in `docker-compose.yml` — remove it if you don't have a GPU.
---
## LLM Backend
-The API talks to LLMs through the OpenAI-compatible `AsyncOpenAI` client — there is **no Anthropic SDK**. The backend is selected with the `LLM_BACKEND` env var:
+Selected with `LLM_BACKEND`:
-| `LLM_BACKEND` | Agent / tool model | Browser-agent model | Notes |
+| `LLM_BACKEND` | Text features (CV parse, fit, cover letter, tailored CV, roadmap) | Browser agent | Needs |
|---|---|---|---|
-| `groq` (compose default) | `meta-llama/llama-4-scout-17b-16e-instruct` | `llama-3.3-70b-versatile` | Requires `GROQ_API_KEY` |
-| `ollama` | `${OLLAMA_MODEL}` (default `qwen3.5:4b`) | `${OLLAMA_MODEL}` | Fully local; no API key |
+| `claude` | `ANTHROPIC_MODEL` (default `claude-sonnet-4-6`) via the `anthropic` SDK | same model via browser-use's `ChatAnthropic` | `ANTHROPIC_API_KEY` |
+| `groq` (compose default) | `GROQ_MODEL` (default `meta-llama/llama-4-scout-17b-16e-instruct`) | same model via browser-use's `ChatGroq` | `GROQ_API_KEY` |
+| `ollama` | `OLLAMA_MODEL` (default `qwen3.5:4b`) | same model via browser-use's `ChatOllama` | local resources |
-Embeddings for the semantic job feed always come from Ollama (`nomic-embed-text`), regardless of backend.
+The Groq browser model must be one browser-use drives with JSON-schema output (see `JsonSchemaModels` in `browser_use/llm/groq/chat.py`). Embeddings always come from Ollama (`nomic-embed-text`), whatever the backend:
-> If you run with `LLM_BACKEND=ollama`, pull the models into the ollama volume first:
-> ```bash
-> docker compose exec ollama ollama pull qwen3.5:4b
-> docker compose exec ollama ollama pull nomic-embed-text
-> ```
+```bash
+docker compose exec ollama ollama pull nomic-embed-text
+docker compose exec ollama ollama pull qwen3.5:4b # only for LLM_BACKEND=ollama
+```
---
@@ -60,39 +59,41 @@ Embeddings for the semantic job feed always come from Ollama (`nomic-embed-text`
### Prerequisites
- Docker + Docker Compose
-- A Groq API key (for the default backend) **or** enough local resources to run Ollama models
-- Adzuna API credentials (free tier) — optional; multiple free job sources and fixture data are used otherwise
+- An API key for your chosen backend (or enough local resources for Ollama)
+- Optional: Adzuna and Brave Search keys for more job sources
-### 1. Clone and configure
+### 1. Configure
```bash
-git clone
-cd Windrush
cp .env.example .env
-# Edit .env (see Environment Variables below)
+# Required secrets:
+openssl rand -hex 32 # → JWT_SECRET
+python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" # → CREDENTIALS_KEY
```
+The API refuses to start without `JWT_SECRET` and `CREDENTIALS_KEY`.
+
### 2. Data files
-The economic index and task-penetration data must be present in `./data/` (bind-mounted read-only at `/data` in the api container):
+The economic index and task-penetration data must be present in `./data/` (bind-mounted read-only at `/data`):
```
data/economic_index.json — Anthropic Economic Index (O*NET occupation exposure scores)
data/task_penetration.csv — O*NET task-level AI penetration scores (~18 000 rows)
```
-### 3. Build and run
+### 3. Run
```bash
docker compose up --build
```
-Open [http://localhost](http://localhost), create an account, and complete onboarding.
+Open [http://localhost](http://localhost), create an account, and complete onboarding. Optionally pre-fill the job store with `docker compose exec api python -m app.job_sync`.
-For local (non-Docker) dev:
+Local (non-Docker) dev:
```bash
-cd api && uvicorn app.main:app --reload # needs a reachable Ollama/Groq + the data files
+cd api && uvicorn app.main:app --reload # needs the env vars above, the data files and a reachable Ollama
cd frontend && npm run dev
```
@@ -102,133 +103,98 @@ cd frontend && npm run dev
| Variable | Required | Description |
|---|---|---|
-| `LLM_BACKEND` | No | `groq` (default in compose) or `ollama` |
-| `GROQ_API_KEY` | If `LLM_BACKEND=groq` | Powers the agent pipeline and browser agent via Groq |
-| `OLLAMA_HOST` | No | Ollama base URL (default `http://ollama:11434` in compose) |
-| `OLLAMA_MODEL` | No | Local model id (default `qwen3.5:4b`) |
-| `EMBEDDING_MODEL` | No | Embedding model for the semantic feed (default `nomic-embed-text`) |
-| `JOB_EMAIL` | For applying | Email credential passed to the browser agent for job-site login |
-| `JOB_PASSWORD` | For applying | Password credential (masked in SSE output) |
-| `ADZUNA_APP_ID` | Optional | Adzuna API id (enables the Adzuna job source) |
-| `ADZUNA_API_KEY` | Optional | Adzuna API key |
-| `ECONOMIC_INDEX_PATH` | No | Path to the economic index JSON (default `/data/economic_index.json`) |
-| `TASK_PENETRATION_PATH` | No | Path to the task penetration CSV (default `/data/task_penetration.csv`) |
-| `JOBS_MCP_URL` | No | Guardrails MCP endpoint (default `http://civic-guardrails:8002/mcp`) |
-| `APP_DATA_PATH` | No | SQLite data dir (default `/appdata` in compose) |
-
-`JOB_EMAIL` / `JOB_PASSWORD` can also be supplied per-application from the persona instead of globally.
+| `JWT_SECRET` | **Yes** | Signs login tokens |
+| `CREDENTIALS_KEY` | **Yes** | Fernet key encrypting stored job-site passwords. Changing it makes saved passwords unreadable (users re-enter them) |
+| `LLM_BACKEND` | No | `claude`, `groq` (compose default) or `ollama` |
+| `ANTHROPIC_API_KEY` / `ANTHROPIC_MODEL` | For `claude` | Key / model id |
+| `GROQ_API_KEY` / `GROQ_MODEL` | For `groq` | Key / model id |
+| `OLLAMA_HOST` / `OLLAMA_MODEL` / `EMBEDDING_MODEL` | No | Ollama URL (compose: `http://ollama:11434`), local model, embedding model |
+| `ADZUNA_APP_ID` / `ADZUNA_API_KEY` | Optional | Enables the Adzuna job source |
+| `BRAVE_SEARCH_API_KEY` | Optional | Enables the Brave `site:` search job source |
+| `ECONOMIC_INDEX_PATH` / `TASK_PENETRATION_PATH` | No | Data file paths (default `/data/...`) |
+| `APP_DATA_PATH` | No | SQLite DBs, PDFs and uploaded CVs (compose: `/appdata`) |
+
+Job-site login details are set per user on the Profile page, not via env vars.
---
## API Endpoints
-All long-running endpoints use **Server-Sent Events (SSE)** so the frontend renders incrementally. Protected endpoints require a `Bearer` JWT from `/login`.
+Long-running endpoints stream **Server-Sent Events (SSE)**. Protected endpoints require a `Bearer` JWT from `/login`.
| Method | Path | Description |
|---|---|---|
| `POST` | `/signup` | Create an account |
| `POST` | `/login` | OAuth2 password form → JWT |
-| `GET` / `PUT` | `/persona` | Read / update the structured persona |
-| `POST` | `/upload` | Parse a CV PDF, merge into the persona, return a `cv_session_id` |
-| `POST` | `/stream` | Run the full agent pipeline → SSE events |
-| `POST` | `/apply` | Trigger the browser application → SSE browser events |
-| `POST` | `/browser-input/{session_id}` | Send click/type/scroll/submit commands to the live session |
-| `GET` | `/browser-stream/{session_id}` | SSE stream of live base64 JPEG frames (CDP screencast) |
-| `GET` | `/jobs` | Paginated, filtered, semantically-ranked job feed |
+| `GET` / `PUT` | `/persona` | Read / update the persona (the job-site password is write-only — GET returns a mask) |
+| `GET` | `/persona/export?format=json\|md\|pdf` | Download the persona |
+| `POST` | `/upload` | Parse a CV PDF and merge it into the persona |
+| `GET` | `/jobs` | Filtered, semantically-ranked job feed; `discovering: true` while background discovery runs |
+| `POST` | `/jobs/from-url` | Scrape a pasted job link into a job object |
| `POST` | `/jobs/save` | Bookmark a job into the tracker as `Saved` |
| `POST` | `/jobs/analyze` | SSE: fit + AI-risk analysis for one job |
-| `POST` | `/jobs/cover-letter` | SSE: generate a tailored cover letter |
-| `POST` | `/jobs/tailored-cv` | SSE: generate a tailored, structured CV |
+| `POST` | `/jobs/cover-letter` | SSE: tailored cover letter |
+| `POST` | `/jobs/tailored-cv` | SSE: tailored, structured CV |
+| `POST` | `/apply` | SSE: browser application (fill → review → submit) |
+| `POST` | `/browser-input/{session_id}` | Click/type/key/scroll/`submit`/`done`/`skip` into the live session |
+| `GET` | `/browser-stream/{session_id}` | SSE: live JPEG frames (CDP screencast) |
| `GET` / `POST` | `/applications` | List / create tracked applications |
-| `PATCH` | `/applications/{id}/status` | Update an application's status + notes |
+| `PATCH` | `/applications/{id}/status` | Update one of *your* applications' status + notes |
| `GET` / `POST` | `/onboarding/status`, `/onboarding/complete` | Onboarding state |
-| `POST` | `/score-skills` | Score AI risk for all persona skills |
-| `POST` | `/careers/roadmap` | SSE: generate a skill-development roadmap |
+| `POST` | `/score-skills` | AI risk for all persona skills |
+| `POST` | `/careers/roadmap` | SSE: skill-development roadmap |
| `POST` | `/documents/pdf` | Render a structured doc (or legacy text) to PDF |
| `POST` | `/documents/preview` | Render a structured doc to HTML for the live preview |
| `GET` | `/documents/{doc_id}/download` | Download a generated PDF |
-| `GET` | `/guardrails/audit` | In-process guardrail audit log |
| `GET` | `/health` | Liveness check |
---
-## Agent Pipeline
-
-The API drives the LLM in an agentic tool-use loop (`api/app/agent.py`). The system prompt enforces this order:
-
-```
-extract_cv_profile → score_ai_risk → search_jobs (+ optional web_search)
- → score_job_fit → generate_skill_roadmap → generate_cover_letter
-```
-
-After the user explicitly approves, the apply phase runs the browser agent. The loop also:
-- nudges the model to continue if it stops early (max 2 nudges),
-- injects a correction if the model hallucinates a tool name.
-
-The full tool set is: `extract_cv_profile`, `score_ai_risk`, `search_jobs`, `web_search` (DuckDuckGo Lite), `score_job_fit`, `generate_cover_letter`, `generate_tailored_cv`, `generate_skill_roadmap`, `apply_to_job`, `lookup_economic_index`.
-
-**Dual-path tool data:**
-- SSE output → PII-redacted (emails, phones, postcodes, NI numbers) and credential-masked.
-- LLM context → unredacted (the agent needs real contact details to fill forms).
-
----
-
## AI Exposure Risk Scoring
-Scores are computed without any neural network — deterministic lookups against O\*NET data, with the LLM only adding human-readable reasoning afterwards.
-
-`score_ai_risk` resolves each skill/title in this order (`agent.py`, `risk_scorer.py`):
+Scores are deterministic lookups against O\*NET data; the LLM only adds human-readable reasoning afterwards. `score_ai_risk` resolves each skill/title in this order (`agent.py`, `risk_scorer.py`):
1. **Curated exposure table** — a hand-tuned map of common tech skills and job titles.
-2. **Keyword task search** — average penetration of all O\*NET tasks containing the term (reliable for multi-word occupational phrases).
+2. **Keyword task search** — average penetration of all O\*NET tasks containing the term.
3. **TF-IDF semantic match** — cosine similarity against ~1 350 non-zero-penetration task descriptions; requires all content words to appear.
4. **O\*NET word-overlap** — stems and noise-strips the title, then matches the occupation with the most shared content words.
5. **Default 0.5** — neutral when no data differentiates.
-Labels: **High ≥ 65 %**, **Medium 35–65 %**, **Low < 35 %**.
-
-**Job fit** (`score_job_fit`) is an LLM semantic analysis returning `fit_score` (0–100), `level_match` (`strong`/`ok`/`reach`), `matched_skills`, `skill_gaps`, and a one-line rationale. **Feed ordering** is done by dense-vector cosine similarity between an interest-weighted persona embedding and each job's embedding.
+Labels: **High ≥ 65 %**, **Medium 35–65 %**, **Low < 35 %**. **Job fit** (`score_job_fit`) is an LLM semantic analysis returning `fit_score` (0–100), `level_match` (`strong`/`ok`/`reach`), `matched_skills`, `skill_gaps` and a one-line rationale.
---
-## Job Discovery
-
-`search_jobs` runs a multi-level cascade (`api/app/job_searcher.py`) and caches results in a local SQLite store (`jobs.db`):
+## Job Feed & Discovery
-1. **Playwright scraping** of curated company career pages.
-2. **Public ATS JSON APIs** — Greenhouse, Ashby, Lever, Workable, SmartRecruiters (60+ companies).
-3. **Brave Search** with `site:` filters.
-4. **Adzuna** paid API (when credentials are set).
-
-If everything is empty, it falls back to `api/app/jobs_fixture.json`. The `/jobs` feed lazily triggers live discovery when local matches run low and persists new jobs (with embeddings) for next time.
+- **Search** (`jobs_db.get_jobs`) filters in SQL: free-text terms (the search box sends them as tags) must appear in title/company/description; role tags (`ml`, `backend`, …) match any; domain tags (`fintech`, `sponsorship`, …) must all match; location uses whole-word aliases (`uk`, `us`, …). Results are ranked by cosine similarity between your persona-plus-search embedding and each job's stored embedding (the 2 000 most recent matches are ranked), and jobs you've applied to or discarded are removed before pagination.
+- **Discovery** (`discovery.py` → `job_searcher.py`) runs when page 1 is thin — in a background **worker process**, so the API stays responsive — with one run per query/location at a time and a 30-minute cooldown. Sources: Playwright scraping of curated career pages; Greenhouse / Ashby / Lever / Workable / SmartRecruiters public APIs (60+ companies); Brave `site:` search; Adzuna. If every source is empty it shows bundled fixture jobs, which are never stored.
+- **Embeddings** are computed only for new jobs, in batched Ollama `/api/embed` calls.
---
## Browser Automation
-The browser agent (`api/app/browser_agent.py`) uses browser-use with the configured backend's model:
+`browser_agent.py` targets browser-use **0.11.13** (pinned — its API changes between minor versions):
-- **No vision** (`use_vision=False`) — the agent reasons from the DOM/accessibility tree, which is faster.
-- **CDP screencast** — live JPEG frames (quality 60, up to ~30 fps, max 1280×800) streamed to the frontend via a separate SSE endpoint, independent of the agent.
-- **File upload** — the CV PDF path is passed as `available_file_paths` so the agent can attach it directly.
-- **Rich task prompt** — persona contact details, skills, education, experience, projects, screening answers, behavioural stories, and dropdown/diversity defaults are compiled into the agent's task.
-- **Interactive fallback + review** — if the agent fails (or before final submit), the frontend becomes a clickable remote control; clicks/types/scrolls are dispatched via Playwright until the user types `submit`/`skip`.
+- **Fill, then review** — the agent is told never to click the final submit button. When it's done you get the live view; `submit` runs a short follow-up agent that clicks it and checks for a confirmation, `done` records that you submitted it yourself, `skip` abandons it. Only a confirmed submission marks the application *Applied*.
+- **Live view & takeover** — CDP screencast frames over a separate SSE endpoint; your clicks/typing/scrolling are replayed onto the page via CDP.
+- **Credentials** — your job-site password is passed as browser-use `sensitive_data`: the model only ever sees `job_password`, and the real value is typed only on the job site's own host over HTTPS. The browser-use "judge" call is disabled because it sees unredacted step history.
+- **Network limits** — the agent's browser can't open raw-IP URLs, `localhost` or the internal service hostnames.
+- **CV upload** — the tailored CV PDF if you generated one, otherwise your uploaded original.
---
-## Guardrails
+## Security & Privacy
-| Layer | Where | What |
-|---|---|---|
-| CV injection check | `guardrails.py` (pre-pipeline) | Regex patterns detect prompt injection in uploaded CV text; blocks before any LLM call |
-| Tool input sanitisation | `guardrails.py` | Validates search queries (injection/HTML), validates `apply_to_job` URLs (`http(s)://`), strips credential fields from all tool inputs |
-| PII redaction (SSE) | `guardrails.py` | Redacts emails, UK/intl phones, postcodes, NI numbers from SSE-streamed tool results/inputs |
-| Credential masking (SSE) | `guardrails.py` | Replaces password/token/secret/api_key values with `***` in SSE events |
-| MCP rate limiting | `civic-guardrails/server.js` | 30 requests/minute per session |
-| MCP search guardrail | `civic-guardrails/server.js` | Blocks injection patterns in `search_jobs` queries |
-| MCP PII scrub + audit | `civic-guardrails/server.js` | Scrubs emails/phones from job responses; logs all MCP requests/responses to `/logs/audit.jsonl` |
-
-Guardrail events surface in the UI (flashing shield badge) and via `GET /guardrails/audit`.
+| Control | Where |
+|---|---|
+| Required `JWT_SECRET` (no default) | `auth.py` |
+| Job-site password encrypted at rest, never returned to the browser | `crypto.py`, `tracker.py`, `main.py` |
+| Persona sent to LLMs is an allowlist — no credentials, DOB, contact details, diversity data or salary | `agent._llm_persona` |
+| Server-side fetches of user-influenced URLs only reach public IPs, re-checked on every redirect | `net_guard.py` |
+| Shared job descriptions can only be refreshed from the stored job's own URL | `main._ensure_full_description` |
+| Application updates scoped to the owning user | `tracker.update_status` |
+| Prompt-injection screening of uploaded CV text | `guardrails.py` |
---
@@ -238,20 +204,7 @@ Guardrail events surface in the UI (flashing shield badge) and via `GET /guardra
**Pages:** `login`, `signup`, `onboarding`, `dashboard`, `jobs` (feed), `jobs/[id]` (analysis + documents + apply), `applications` (tracker), `careers` (risk + roadmap), `profile` (persona editor).
-**Key components:**
-
-| Component | Purpose |
-|---|---|
-| `AppShell` / `Sidebar` | Navigation shell (hidden on auth/onboarding pages) |
-| `RiskRadar` | AI exposure % per skill |
-| `JobCard` / `JobList` | Ranked job cards with badges |
-| `DocEditor` | Structured CV/cover-letter editor with live HTML preview |
-| `CoverLetter` | Cover-letter review + job-site credential entry |
-| `BrowserView` | Clickable live browser screencast; clicks sent to `/browser-input` |
-| `AgentLog` | Real-time tool calls, results, guardrail events, browser steps |
-| `SkillRoadmap` | Timeline of skill recommendations |
-| `GuardrailBadge` | Flashes when a guardrail fires |
-| `ApplicationTracker` | Status lifecycle management |
+**Components:** `AppShell` / `Sidebar` (navigation), `JobCard`, `DocEditor` (structured CV/cover-letter editor with live preview), `BrowserView` (clickable live screencast).
---
@@ -259,31 +212,28 @@ Guardrail events surface in the UI (flashing shield badge) and via `GET /guardra
| File | Description |
|---|---|
-| `data/economic_index.json` | ~3 000 O\*NET occupation codes → `overall_exposure` (0–1). Source: Anthropic Economic Index. |
-| `data/task_penetration.csv` | ~18 000 O\*NET task descriptions → `penetration` score (TF-IDF + keyword search). |
-| `api/app/jobs_fixture.json` | Mock job listings used as a last-resort fallback. |
-| `applications.db` (SQLite, `/appdata`) | `users` (bcrypt passwords, persona JSON, onboarding flag) + `applications`. |
-| `jobs.db` (SQLite, `/appdata`) | Cached jobs with semantic vectors; populated lazily or via `python -m app.job_sync`. |
+| `data/economic_index.json` | 756 O\*NET occupations → `overall_exposure` (0–1). Source: Anthropic Economic Index. |
+| `data/task_penetration.csv` | ~18 000 O\*NET task descriptions → `penetration` score. |
+| `api/app/jobs_fixture.json` | Mock job listings shown only when every live source is empty. |
+| `applications.db` (SQLite, `/appdata`) | `users` (bcrypt password hashes, persona JSON, encrypted job-site password, onboarding flag) + `applications`. |
+| `jobs.db` (SQLite, `/appdata`) | Cached jobs with semantic vectors; filled by discovery or `python -m app.job_sync`. |
+| `/appdata/pdfs`, `/appdata/cvs` | Generated PDFs; each user's uploaded CV. |
---
-## Models in Use
-
-| Use case | `groq` backend | `ollama` backend |
-|---|---|---|
-| Agent loop + internal tools (CV parse, fit, cover letter, roadmap, tailored CV) | `meta-llama/llama-4-scout-17b-16e-instruct` | `qwen3.5:4b` (`OLLAMA_MODEL`) |
-| Browser automation | `llama-3.3-70b-versatile` | `qwen3.5:4b` (`OLLAMA_MODEL`) |
-| Embeddings (semantic feed) | `nomic-embed-text` (Ollama) | `nomic-embed-text` (Ollama) |
+## Development
----
+**Tests** (`api/tests/`):
-## Development Notes
+```bash
+cd api
+pip install -r requirements.txt -r requirements-dev.txt
+pytest # fast suite — no network, LLM or browser
+pytest -m browser # real headless Chromium (needs: playwright install chromium)
+```
-**Switching backend:** set `LLM_BACKEND=ollama` (and pull the models) or `LLM_BACKEND=groq` (and set `GROQ_API_KEY`).
+CI (`.github/workflows/ci.yml`) runs both, plus a frontend type-check and build.
-**Adding a new tool:**
-1. Add the definition to `TOOLS` in `agent.py` (`name`, `description`, `input_schema`).
-2. Add a handler branch in `execute_tool()`.
-3. If the tool processes user data, add it to `_PII_REDACT_TOOLS` in `guardrails.py`.
+**Switching backend:** set `LLM_BACKEND` and the matching key (see above).
-**Document templates:** structured CV/cover-letter rendering lives in `api/app/doc_render.py` + `api/app/templates/` (`registry.py`, `classic_cv.html`, `classic_letter.html`, `base.css`), rendered with Jinja2 → WeasyPrint (fpdf2 is the fallback).
+**Document templates:** `api/app/doc_render.py` + `api/app/templates/` (`registry.py`, `classic_cv.html`, `classic_letter.html`, `base.css`), rendered with Jinja2 → WeasyPrint (fpdf2 fallback).
diff --git a/api/app/agent.py b/api/app/agent.py
index c1cb9b6..618e5a9 100644
--- a/api/app/agent.py
+++ b/api/app/agent.py
@@ -4,33 +4,30 @@
import os
import re
import time
+from contextlib import aclosing
from html.parser import HTMLParser
from typing import AsyncGenerator
-import httpx
from openai import AsyncOpenAI
+from . import http_client
from . import tracker
-from .cv_parser import extract_text
-from .risk_scorer import lookup_onet, lookup_by_title, ECONOMIC_INDEX
-from .job_proxy import search_jobs
+from .risk_scorer import lookup_by_title
from .browser_agent import apply_with_browser
-from .guardrails import (
- sanitise_tool_input,
- redact_pii_from_result,
- redact_pii_from_input,
- redact_credentials_from_input,
- GuardrailViolation,
-)
logger = logging.getLogger("windrush.agent")
# ── LLM backend ───────────────────────────────────────────────────────────────
-# Switch between Ollama (local) and Groq (cloud) via LLM_BACKEND env var.
-# LLM_BACKEND=ollama → local Gemma 4 via Ollama
-# LLM_BACKEND=groq → Groq cloud (original)
+# Switch backends via the LLM_BACKEND env var:
+# LLM_BACKEND=ollama → local model via Ollama (OpenAI-compat client)
+# LLM_BACKEND=groq → Groq cloud (OpenAI-compat client)
+# LLM_BACKEND=claude → Anthropic Claude via the native `anthropic` SDK
+# Embeddings always come from Ollama (nomic-embed-text) regardless of backend.
_BACKEND = os.environ.get("LLM_BACKEND", "ollama").lower()
+_IS_CLAUDE = _BACKEND == "claude"
+client = None # OpenAI-compat client (groq/ollama only)
+_anthropic = None # native Anthropic async client (claude only)
if _BACKEND == "groq":
client = AsyncOpenAI(
@@ -41,8 +38,17 @@
# … in content and need think-stripping + higher max_tokens, and
# smaller-tier models may have low TPM limits that reject full-persona payloads.
_GROQ_MODEL = os.environ.get("GROQ_MODEL", "meta-llama/llama-4-scout-17b-16e-instruct")
- AGENT_MODEL = _GROQ_MODEL
LLM_MODEL = _GROQ_MODEL
+elif _BACKEND == "claude":
+ # Native Anthropic SDK (not an OpenAI-compat shim). Sonnet 4.6 by default;
+ # override with ANTHROPIC_MODEL. No Groq-style TPM wall on the standard tier.
+ import anthropic
+ _anthropic = anthropic.AsyncAnthropic(
+ api_key=os.environ.get("ANTHROPIC_API_KEY", ""),
+ timeout=120.0,
+ )
+ _CLAUDE_MODEL = os.environ.get("ANTHROPIC_MODEL", "claude-sonnet-4-6")
+ LLM_MODEL = _CLAUDE_MODEL
else:
# Ollama — local Qwen 3.5 4B
_OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://ollama:11434")
@@ -51,32 +57,8 @@
api_key="ollama", # Ollama ignores the key but requires a value
base_url=f"{_OLLAMA_HOST}/v1",
)
- AGENT_MODEL = _OLLAMA_MODEL
LLM_MODEL = _OLLAMA_MODEL
-SYSTEM_PROMPT = """You are Windrush, an AI career transition advisor helping workers navigate the impact of AI on their careers.
-
-AVAILABLE TOOLS — use these exact names, spelled exactly as shown:
- extract_cv_profile | score_ai_risk | search_jobs | web_search | score_job_fit | generate_skill_roadmap | generate_cover_letter | apply_to_job
-
-Work through this pipeline in order, passing data between tools explicitly:
-
-1. extract_cv_profile(cv_text) → returns a basic profile object (name, skills, job_titles, location).
- NOTE: For logged-in users, you will be provided with a comprehensive 'persona' object. Use that 'persona' in all subsequent steps.
-2. score_ai_risk(skills=[...all skills and job_titles from the profile or persona...])
-3. search_jobs(query=, location=)
- OPTIONAL: call web_search with a targeted site: query to supplement results.
- Merge any additional jobs into the list before passing to score_job_fit.
-4. score_job_fit(jobs=[...], persona={...the comprehensive persona object...})
-5. generate_skill_roadmap(skill_risks=[...], persona={...}, target_job_title=)
-6. generate_cover_letter(job={...}, persona={...})
-
-CRITICAL RULES:
-- After each tool returns a result, immediately call the next tool in the pipeline. Do NOT write any text, summary, or response to the user between tool calls.
-- Always pass the actual data objects from previous tool results into subsequent tool calls. Never call a tool with empty arguments.
-- After generate_cover_letter completes, stop. Do NOT call apply_to_job — the user must explicitly approve first."""
-
-
_HARDCODED_EXPOSURE: dict[str, float] = {
# Programming languages
"python": 0.78, "javascript": 0.71, "typescript": 0.68, "java": 0.69,
@@ -114,182 +96,18 @@
"devops engineer": 0.55, "platform engineer": 0.53,
}
-TOOLS = [
- {
- "name": "extract_cv_profile",
- "description": "Parse CV text into structured profile: name, skills, job titles, experience years, location.",
- "input_schema": {
- "type": "object",
- "properties": {
- "cv_text": {"type": "string", "description": "Raw CV text to parse"},
- },
- "required": ["cv_text"],
- },
- },
- {
- "name": "score_ai_risk",
- "description": "Look up AI exposure scores for a list of skills or job titles using the Economic Index.",
- "input_schema": {
- "type": "object",
- "properties": {
- "skills": {
- "type": "array",
- "items": {"type": "string"},
- "description": "List of skills or occupation titles to score",
- },
- },
- "required": ["skills"],
- },
- },
- {
- "name": "search_jobs",
- "description": "Search for job listings matching a query in a given location.",
- "input_schema": {
- "type": "object",
- "properties": {
- "query": {"type": "string", "description": "Job search query"},
- "location": {"type": "string", "description": "Location e.g. 'London'"},
- },
- "required": ["query", "location"],
- },
- },
- {
- "name": "web_search",
- "description": (
- "Search the web for job listings or company information. "
- "Use site: filters to target specific job boards, e.g. "
- "'site:jobs.ashbyhq.com \"Junior AI Engineer\" OR \"Graduate ML\"'. "
- "Returns a list of {title, url, snippet} results. "
- "Call this after search_jobs to supplement with targeted queries."
- ),
- "input_schema": {
- "type": "object",
- "properties": {
- "query": {
- "type": "string",
- "description": (
- "Search query. Use site: for job boards, e.g. "
- "'site:jobs.ashbyhq.com \"Graduate\" \"AI Engineer\"'"
- ),
- },
- },
- "required": ["query"],
- },
- },
- {
- "name": "score_job_fit",
- "description": "Score and rank a list of jobs against a user persona. Returns jobs sorted by composite score.",
- "input_schema": {
- "type": "object",
- "properties": {
- "jobs": {
- "type": "array",
- "items": {"type": "object"},
- "description": "List of job objects from search_jobs",
- },
- "persona": {
- "type": "object",
- "description": "Structured User Persona",
- },
- },
- "required": ["jobs", "persona"],
- },
- },
- {
- "name": "generate_cover_letter",
- "description": "Generate a personalised cover letter for a specific job, grounded in the user persona.",
- "input_schema": {
- "type": "object",
- "properties": {
- "job": {"type": "object", "description": "Target job object"},
- "persona": {"type": "object", "description": "Structured User Persona"},
- "tone": {
- "type": "string",
- "enum": ["professional", "enthusiastic", "concise"],
- "description": "Tone of the cover letter",
- },
- },
- "required": ["job", "persona"],
- },
- },
- {
- "name": "apply_to_job",
- "description": "Submit a job application. Only call this after explicit user approval.",
- "input_schema": {
- "type": "object",
- "properties": {
- "job_id": {"type": "string"},
- "cover_letter": {"type": "string"},
- "persona": {"type": "object"},
- },
- "required": ["job_id", "cover_letter", "persona"],
- },
- },
- {
- "name": "generate_skill_roadmap",
- "description": "Generate a prioritised skill development roadmap tailored to the candidate's industry and career goals.",
- "input_schema": {
- "type": "object",
- "properties": {
- "skill_risks": {
- "type": "array",
- "items": {"type": "object"},
- "description": "List of scored skill risks from score_ai_risk",
- },
- "persona": {
- "type": "object",
- "description": "Structured User Persona",
- },
- "target_job_title": {
- "type": "string",
- "description": "Title of the job the candidate is targeting",
- },
- },
- "required": ["skill_risks", "persona"],
- },
- },
- {
- "name": "lookup_economic_index",
- "description": "Look up a specific ONET SOC code in the Economic Index for raw exposure data.",
- "input_schema": {
- "type": "object",
- "properties": {
- "onet_code": {"type": "string", "description": "ONET SOC code e.g. '15-1252.00'"},
- },
- "required": ["onet_code"],
- },
- },
- {
- "name": "generate_tailored_cv",
- "description": "Generate a tailored CV (structured JSON CVDoc) for a specific job, emphasising the most relevant experience and skills from the user persona.",
- "input_schema": {
- "type": "object",
- "properties": {
- "job": {"type": "object", "description": "Target job object with title, company, description"},
- "persona": {"type": "object", "description": "Structured User Persona"},
- "template_id": {"type": "string", "description": "Document template id (default 'classic')"},
- },
- "required": ["job", "persona"],
- },
- },
-]
-
-
-GROQ_TOOLS = [
- {
- "type": "function",
- "function": {
- "name": t["name"],
- "description": t["description"],
- "parameters": t["input_schema"],
- },
- }
- for t in TOOLS
-]
-
-
async def _llm(system: str, user: str, max_tokens: int = 2048) -> str:
"""Simple single-turn LLM call for internal tool use (CV parsing, cover letter, roadmap)."""
+ if _IS_CLAUDE:
+ # Native Anthropic Messages API: system is a top-level param; reply is a list
+ # of content blocks (thinking lives in separate blocks, so the text is clean).
+ resp = await _anthropic.messages.create(
+ model=LLM_MODEL,
+ max_tokens=max_tokens,
+ system=system,
+ messages=[{"role": "user", "content": user}],
+ )
+ return "".join(b.text for b in resp.content if b.type == "text")
resp = await client.chat.completions.create(
model=LLM_MODEL,
max_tokens=max_tokens,
@@ -410,6 +228,36 @@ def _recency_key(entry: dict) -> tuple[int, int]:
return (year, month)
+# core_info fields an LLM may see. Everything else — credentials, DOB, email, phone, postal
+# address — is identity/contact data the writing and scoring tasks don't need; documents get
+# contact details from the persona directly, never from the model.
+_LLM_CORE_FIELDS = (
+ "first_name", "last_name", "preferred_name", "city", "country",
+ "linkedin", "github", "portfolio", "website",
+ "visa_status", "visa_type", "right_to_work_uk", "require_sponsorship", "security_clearance",
+)
+
+
+def _llm_persona(persona: dict) -> dict:
+ """
+ The persona as sent to third-party LLMs: an allowlist of core_info, no diversity / equal-
+ opportunities data, and no salary figure. Use this for every prompt that embeds the persona.
+ """
+ core = persona.get("core_info", {}) or {}
+ slim = {k: v for k, v in persona.items() if k not in ("core_info", "diversity", "screening")}
+ slim["core_info"] = {k: core[k] for k in _LLM_CORE_FIELDS if k in core}
+ screening = {k: v for k, v in (persona.get("screening") or {}).items() if k != "salary_canonical"}
+ if screening:
+ slim["screening"] = screening
+ return slim
+
+
+def _cert_date(raw: str) -> str:
+ """Render the profile's value ('YYYY-MM') as MM/YYYY; pass anything else through."""
+ m = re.fullmatch(r"\s*((?:19|20)\d{2})-(0[1-9]|1[0-2])\s*", raw or "")
+ return f"{m.group(2)}/{m.group(1)}" if m else (raw or "").strip()
+
+
def _persona_to_cvdoc(persona: dict) -> dict:
"""
Build a complete CVDoc straight from the persona — used both as the structured
@@ -468,8 +316,12 @@ def _persona_to_cvdoc(persona: dict) -> dict:
"tech": p.get("technologies", []) if isinstance(p.get("technologies"), list) else [],
"link": p.get("url", ""),
})
+ # Persona certs use issuing_organization / issue_date (models.Certification); the
+ # CVDoc uses issuer / year. Older payloads may already carry the CVDoc names.
certifications = [
- {"name": c.get("name", ""), "issuer": c.get("issuer", ""), "year": c.get("year", "")}
+ {"name": c.get("name", ""),
+ "issuer": c.get("issuing_organization", "") or c.get("issuer", ""),
+ "year": _cert_date(c.get("issue_date", "") or c.get("year", ""))}
for c in persona.get("certifications", [])
]
return {
@@ -682,19 +534,6 @@ def _detect_archetype(job_title: str, description: str) -> str:
return max(scores, key=lambda k: scores[k])
-def _extract_differentiation_hook(cv_profile: dict, job_description: str) -> str:
- """Find a distinctive item from the candidate's experience not mentioned in the JD."""
- jd_lower = job_description.lower()
- for exp in cv_profile.get("experience", []):
- summary = exp.get("summary", "") + " " + exp.get("employer", "")
- # Look for capitalised proper nouns / specific tech terms not in JD
- tokens = [w.strip(".,();") for w in summary.split() if len(w) > 3 and w[0].isupper()]
- for token in tokens:
- if token.lower() not in jd_lower and len(token) > 4:
- return f"Notable experience with {token} that directly applies to this role."
- return ""
-
-
class _DDGLiteParser(HTMLParser):
"""
Parser for DuckDuckGo Lite (https://lite.duckduckgo.com/lite/) HTML results.
@@ -754,7 +593,7 @@ async def _ddg_search(query: str, max_results: int = 15) -> list[dict]:
Returns a list of {title, url, snippet} dicts.
"""
try:
- async with httpx.AsyncClient(
+ async with http_client.async_client(
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -868,37 +707,6 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
except json.JSONDecodeError:
return {"skill_risks": raw_results}
- elif name == "search_jobs":
- jobs = await search_jobs(tool_input.get("query", ""), tool_input.get("location", "London"))
- slim_jobs = [
- {k: v for k, v in j.items() if k not in ("salary_min", "salary_max")}
- for j in jobs[:10]
- ]
- return {"jobs": slim_jobs, "count": len(slim_jobs)}
-
- elif name == "web_search":
- query = tool_input.get("query", "")
- results = await _ddg_search(query)
- # Shape into lightweight job-like dicts the agent can merge with search_jobs output
- jobs = []
- for r in results:
- title_raw = r.get("title", "")
- # Try to split "Title @ Company" / "Title at Company"
- m = re.search(r"(.+?)(?:\s*[@|]\s*|\s+at\s+)(.+?)$", title_raw, re.I)
- title = m.group(1).strip() if m else title_raw
- company = m.group(2).strip() if m else "Unknown"
- if len(title) < 4:
- continue
- jobs.append({
- "job_id": f"ws-{abs(hash(r['url'])) % 10**8}",
- "title": title,
- "company": company,
- "location": "See listing",
- "description": r.get("snippet", "")[:500],
- "url": r.get("url", ""),
- })
- return {"results": results, "jobs": jobs, "count": len(jobs)}
-
elif name == "score_job_fit":
jobs = tool_input.get("jobs", [])
persona = tool_input.get("persona", {})
@@ -924,7 +732,7 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
"Output ONLY the JSON, no markdown."
),
user=(
- f"Candidate Persona: {json.dumps(persona)}\n\n"
+ f"Candidate Persona: {json.dumps(_llm_persona(persona))}\n\n"
f"Jobs to Analyse: {json.dumps(jobs[:5])}"
),
max_tokens=2048,
@@ -1031,7 +839,7 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
data = await _llm_json(
system=system,
user=(
- f"CANDIDATE PERSONA (JSON): {json.dumps(persona)}\n\n"
+ f"CANDIDATE PERSONA (JSON): {json.dumps(_llm_persona(persona))}\n\n"
f"JOB: {job.get('title')} at {job.get('company')}\n"
f"DESCRIPTION: {job.get('description', '')[:2500]}"
+ (f"\n\nMOST DOMAIN-RELEVANT PROJECTS for this role (use the best fit as your main 'Why me' "
@@ -1088,13 +896,6 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
"archetype": archetype,
}
- elif name == "apply_to_job":
- return {
- "status": "submitted",
- "job_id": tool_input.get("job_id"),
- "message": "Application submitted successfully. Good luck!",
- }
-
elif name == "generate_skill_roadmap":
skill_risks = tool_input.get("skill_risks", [])
persona = tool_input.get("persona", {})
@@ -1163,10 +964,6 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
data = {}
return {"status": "generated", "items": data.get("items", [])}
- elif name == "lookup_economic_index":
- onet_code = tool_input.get("onet_code", "")
- return lookup_onet(onet_code)
-
elif name == "generate_tailored_cv":
job = tool_input.get("job", {})
persona = tool_input.get("persona", {})
@@ -1176,7 +973,9 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
# Feed the model the persona twin's FULL project data (problem statement,
# quantified outcomes and technologies) instead of the thin 2-sentence
# projection, so it can both SELECT by relevance and DESCRIBE with real metrics.
- llm_cv = {**base_cv, "projects": _rank_projects_by_relevance(job, _persona_projects_for_llm(persona))}
+ # Contact details are restored from base_cv after generation, so the model never needs them.
+ llm_cv = {**base_cv, "contact": {},
+ "projects": _rank_projects_by_relevance(job, _persona_projects_for_llm(persona))}
level = _seniority(persona)
if level == "junior":
level_guidance = (
@@ -1237,7 +1036,7 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
"Leave 'headline' EMPTY — do not add a title line or echo the target job's role under the "
"candidate's name. "
"Use ONLY facts present in the provided candidate data — do NOT invent employers, dates, "
- "metrics or skills. Keep every contact field exactly as given."
+ "metrics or skills. Leave 'contact' empty — it is filled in separately."
),
user=(
f"TARGET JOB: {job.get('title','')} at {job.get('company','')}\n"
@@ -1278,141 +1077,11 @@ async def execute_tool(name: str, tool_input: dict) -> dict:
return {"error": f"Unknown tool: {name}"}
-async def _chat(messages: list, tools: list):
- """Call LLM via OpenAI-compatible API (Ollama or Groq)."""
- all_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages
- # Ollama local inference can be slow on large models — use a longer timeout (600s)
- timeout = 600.0 if _BACKEND == "ollama" else 90.0
- return await asyncio.wait_for(
- client.chat.completions.create(
- model=AGENT_MODEL,
- max_tokens=4096,
- messages=all_messages,
- tools=tools,
- ),
- timeout=timeout,
- )
-
-
def _sse(event_type: str, data: dict) -> str:
payload = {"type": event_type, "timestamp": time.time(), **data}
return f"data: {json.dumps(payload)}\n\n"
-async def run_pipeline(user_id: str, cv_text: str, location: str = "London") -> AsyncGenerator[str, None]:
- """Yields SSE-formatted strings for each agent step."""
- persona_data = tracker.get_user_persona(user_id)
- persona_str = json.dumps(persona_data, indent=2)
-
- messages: list[dict] = [
- {
- "role": "user",
- "content": (
- f"Please analyse this CV and find suitable jobs in {location}.\n\n"
- f"I have a comprehensive persona already stored for this user:\n{persona_str}\n\n"
- f"CV Text for reference (if it has new info):\n{cv_text[:3000]}"
- ),
- },
- ]
-
- yield _sse("start", {"message": "Pipeline started with your Persistent Persona"})
-
- _cover_letter_done = False
- _nudge_count = 0 # guard against infinite nudge loops
- _MAX_NUDGES = 2
-
- while True:
- try:
- response = await _chat(messages, GROQ_TOOLS)
- except asyncio.TimeoutError:
- yield _sse("done", {"message": "Request timed out — please try again."})
- return
-
- message = response.choices[0].message
- finish_reason = response.choices[0].finish_reason
-
- # Emit any text content
- if message.content and message.content.strip():
- yield _sse("text", {"text": message.content})
-
- tool_calls = message.tool_calls or []
-
- # Append assistant turn (must include tool_calls if present)
- assistant_msg: dict = {"role": "assistant", "content": message.content}
- if tool_calls:
- assistant_msg["tool_calls"] = [
- {
- "id": tc.id,
- "type": "function",
- "function": {"name": tc.function.name, "arguments": tc.function.arguments},
- }
- for tc in tool_calls
- ]
- messages.append(assistant_msg)
-
- for tc in tool_calls:
- tool_name = tc.function.name
- tool_input = json.loads(tc.function.arguments)
-
- sse_input, _ = redact_credentials_from_input(tool_name, tool_input)
- sse_input, _ = redact_pii_from_input(tool_name, sse_input)
- yield _sse("tool_call", {"tool_name": tool_name, "tool_input": sse_input})
-
- try:
- safe_input = sanitise_tool_input(tool_name, tool_input)
- except GuardrailViolation as e:
- yield _sse("guardrail", {"check": e.check, "detail": e.detail, "tool_name": tool_name, "fired": True})
- result = {"error": f"Guardrail blocked this tool call: {e.detail}"}
- yield _sse("tool_result", {"tool_name": tool_name, "result": result})
- messages.append({"role": "tool", "tool_call_id": tc.id, "content": json.dumps(result)})
- continue
-
- result = await execute_tool(tool_name, safe_input)
-
- # Unknown tool — model hallucinated a name. Inject a directive correction
- # so it retries with the right name rather than silently stopping.
- if isinstance(result, dict) and result.get("error", "").startswith("Unknown tool"):
- correction = (
- f"'{tool_name}' is not a valid tool name. "
- "Available tools: extract_cv_profile, score_ai_risk, search_jobs, "
- "web_search, score_job_fit, generate_skill_roadmap, generate_cover_letter. "
- "Call the correct tool now to continue the pipeline."
- )
- result["correction"] = correction
- yield _sse("tool_result", {"tool_name": tool_name, "result": result})
- messages.append({"role": "tool", "tool_call_id": tc.id, "content": json.dumps(result)})
- continue
-
- sse_result, pii_fired = redact_pii_from_result(tool_name, result)
- if pii_fired:
- yield _sse("guardrail", {"check": "pii_redact", "tool_name": tool_name, "fired": True, "detail": "PII removed from display"})
- yield _sse("tool_result", {"tool_name": tool_name, "result": sse_result})
-
- messages.append({"role": "tool", "tool_call_id": tc.id, "content": json.dumps(result)})
-
- if tool_name == "generate_cover_letter":
- _cover_letter_done = True
- yield _sse("done", {"message": "Pipeline complete"})
- return
-
- if finish_reason == "stop" or not tool_calls:
- if not _cover_letter_done and _nudge_count < _MAX_NUDGES:
- # Model stopped before completing the pipeline — nudge it to continue
- _nudge_count += 1
- nudge = (
- "Continue the pipeline. You must still call the remaining tools in order: "
- "score_job_fit (pass the jobs array and cv_profile), "
- "generate_skill_roadmap (pass skill_risks and cv_profile), "
- "generate_cover_letter (pass the top job and cv_profile). "
- "Call the next tool now."
- )
- messages.append({"role": "user", "content": nudge})
- yield _sse("text", {"text": f"[Resuming pipeline — step {_nudge_count}]"})
- continue
- yield _sse("done", {"message": "Pipeline complete"})
- break
-
-
async def run_apply(
user_id: str,
job_id: str,
@@ -1432,19 +1101,26 @@ async def run_apply(
yield _sse("start", {"message": "Starting browser application with your Persona…", "session_id": session_id})
+ submitted = False
if job_url and instruction_queue is not None:
- async for step in apply_with_browser(
+ # aclosing: close the browser generator (its finally kills Chromium) as soon as we
+ # stop iterating — on `break` below or when our own consumer disconnects.
+ async with aclosing(apply_with_browser(
job_url, persona, cover_letter, instruction_queue, frame_queue,
job_email=job_email, job_password=job_password, cv_path=cv_path,
- ):
- event_type = "browser_blocked" if step.get("blocked") else "browser_action"
- yield _sse(event_type, {
- "action": step.get("action", ""),
- "screenshot": step.get("screenshot"),
- "reason": step.get("reason"),
- "interactive": step.get("interactive", False),
- })
- if step.get("done"):
- break
+ )) as steps:
+ async for step in steps:
+ event_type = "browser_blocked" if step.get("blocked") else "browser_action"
+ yield _sse(event_type, {
+ "action": step.get("action", ""),
+ "screenshot": step.get("screenshot"),
+ "reason": step.get("reason"),
+ "interactive": step.get("interactive", False),
+ })
+ if step.get("done"):
+ submitted = bool(step.get("submitted"))
+ break
- yield _sse("done", {"message": "Done"})
+ # `submitted` is the only signal the caller should treat as "application sent";
+ # `done` alone also fires on skip / cancel / timeout / browser errors.
+ yield _sse("done", {"message": "Done", "submitted": submitted})
diff --git a/api/app/auth.py b/api/app/auth.py
index 40cfac7..2fbb6fe 100644
--- a/api/app/auth.py
+++ b/api/app/auth.py
@@ -11,7 +11,10 @@
from . import tracker
# ── CONFIGURATION ─────────────────────────────────────────────────────────────
-SECRET_KEY = os.environ.get("JWT_SECRET", "change-this-in-production-use-openssl-rand-hex-32")
+SECRET_KEY = os.environ.get("JWT_SECRET", "")
+if not SECRET_KEY:
+ # No insecure fallback: a default secret lives in the repo and would let anyone forge tokens.
+ raise RuntimeError("JWT_SECRET is not set — generate one with `openssl rand -hex 32` and add it to .env")
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7 # 7 days
diff --git a/api/app/browser_agent.py b/api/app/browser_agent.py
index 48d3b78..ae4095d 100644
--- a/api/app/browser_agent.py
+++ b/api/app/browser_agent.py
@@ -1,144 +1,94 @@
+"""
+Autonomous job-application filling with browser-use (pinned to 0.11.x — see requirements.txt).
+
+Flow: the agent fills the form and STOPS before the final submit → the user reviews it in the
+live view (and can take over with clicks / typing) → 'submit' has a short follow-up agent click
+the final button, 'done' means the user submitted it themselves, 'skip' abandons it.
+
+browser-use 0.11 is CDP-native (no Playwright, no LangChain): LLMs must be its own
+`browser_use` Chat* wrappers, and everything here talks to Chrome through the session's CDP client.
+"""
import asyncio
import base64
import json
+import logging
import os
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Awaitable, Callable
-from playwright.async_api import Page
+logger = logging.getLogger("windrush.browser_agent")
_BACKEND = os.environ.get("LLM_BACKEND", "ollama").lower()
_OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://ollama:11434")
-_OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "qwen3.5:9b")
-
-try:
- from browser_use import Agent, BrowserSession, BrowserProfile
-
- if _BACKEND == "groq":
- import json as _json
- import re as _re
- import typing as _typing
- from langchain_groq import ChatGroq as _BaseChatGroq
- from langchain_core.messages import (
- SystemMessage as _SystemMessage,
- HumanMessage as _HumanMessage,
- AIMessage as _AIMessage,
- )
+_OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "qwen3.5:4b")
+
+# Viewport the agent browses at. BrowserView.tsx maps clicks on the live frame onto these
+# exact dimensions (BROWSER_W / BROWSER_H) — keep the two in sync.
+VIEWPORT = {"width": 1280, "height": 800}
+
+# browser-use defaults to 500 steps; cap the LLM spend per application.
+_MAX_FILL_STEPS = 40
+_MAX_SUBMIT_STEPS = 4
+
+# Navigation the agent's browser must never reach: it runs inside the api container, next to
+# the internal services. Raw-IP URLs are blocked separately (BrowserProfile.block_ip_addresses).
+# 'http*://' because browser-use patterns without a scheme only match https.
+_BLOCK_IP_ADDRESSES = True
+_PROHIBITED_HOSTS = [
+ f"http*://{h}" for h in (
+ "localhost", "*.localhost", "host.docker.internal", "metadata.google.internal",
+ # docker-compose service names (see docker-compose.yml)
+ "api", "frontend", "nginx", "ollama",
+ )
+]
- def _to_lc_messages(messages):
- """Convert browser_use message types to standard LangChain messages."""
- result = []
- for msg in messages:
- cls_name = type(msg).__name__
- content = getattr(msg, "content", "")
- if cls_name == "SystemMessage":
- result.append(_SystemMessage(content=content))
- elif cls_name == "HumanMessage":
- result.append(_HumanMessage(content=content))
- elif cls_name == "AIMessage":
- result.append(_AIMessage(content=content))
- else:
- role = getattr(msg, "role", "user")
- if role == "system":
- result.append(_SystemMessage(content=content))
- elif role == "assistant":
- result.append(_AIMessage(content=content))
- else:
- result.append(_HumanMessage(content=content))
- return result
-
- def _parse_with_defaults(model_class, text: str):
- """
- Extract JSON from LLM text and fill in defaults for any missing fields
- before constructing the pydantic model. Handles the case where the
- model omits required string fields (evaluation_previous_goal, etc.).
- """
- m = _re.search(r'\{.*\}', text, _re.DOTALL)
- data = _json.loads(m.group()) if m else {}
-
- for name, fi in model_class.model_fields.items():
- if name in data:
- continue
- ann = fi.annotation
- origin = getattr(ann, '__origin__', None)
- if origin is list or ann is list:
- data[name] = []
- elif ann is str or ann == type(""):
- data[name] = ""
- elif ann is int:
- data[name] = 0
- elif ann is bool:
- data[name] = False
- else:
- data[name] = None
-
- return model_class.model_validate(data)
-
- class _BrowserLLM:
- """
- Plain adapter around ChatGroq for browser-use 0.11.x.
-
- browser-use calls:
- response = await llm.ainvoke(messages, output_format=AgentOutput)
- parsed = response.completion
-
- We use Groq's json_object mode (no tool-call schema validation) and
- fill in sensible defaults for any fields the model omits.
- """
-
- provider = "groq"
-
- def __init__(self, model: str, groq_api_key: str):
- self.model = model
- self._inner = _BaseChatGroq(model=model, groq_api_key=groq_api_key)
-
- async def ainvoke(self, messages, output_format=None, **kwargs):
- kwargs.pop("session_id", None)
- lc_messages = _to_lc_messages(messages)
-
- if output_format is not None:
- # json_object mode: Groq validates JSON syntax only, not schema.
- # The model already knows the expected structure from the system prompt.
- llm = self._inner.bind(response_format={"type": "json_object"})
- response = await llm.ainvoke(lc_messages)
- content = getattr(response, "content", str(response))
- completion = _parse_with_defaults(output_format, content)
- return type("_Resp", (), {"completion": completion})()
-
- return await self._inner.ainvoke(lc_messages, **kwargs)
-
- def __getattr__(self, name: str):
- return getattr(self._inner, name)
-
- def _make_browser_llm():
- return _BrowserLLM(
- model="llama-3.3-70b-versatile",
- groq_api_key=os.environ.get("GROQ_API_KEY", ""),
- )
- else:
- from langchain_ollama import ChatOllama as _ChatOllama
+# How long the review / takeover phase waits for the user's next command. Must stay under
+# nginx's proxy_read_timeout (600s) for /api/apply, which sees no bytes while we wait.
+_INPUT_TIMEOUT_S = 300
- def _make_browser_llm():
- return _ChatOllama(
- model=_OLLAMA_MODEL,
- base_url=_OLLAMA_HOST,
- )
+try:
+ from browser_use import Agent, BrowserProfile, BrowserSession
+ from browser_use.actor.page import Page as _ActorPage
BROWSER_USE_AVAILABLE = True
except ImportError:
BROWSER_USE_AVAILABLE = False
- def _make_browser_llm(): # noqa: F811
- return None
+
+def _make_browser_llm():
+ """The browser-use chat model for the configured backend (its own wrappers, not LangChain)."""
+ if _BACKEND == "groq":
+ from browser_use import ChatGroq
+ # Same model as the main agent. browser-use drives Groq via json_schema output, so this
+ # must be a model listed in browser_use.llm.groq.chat.JsonSchemaModels (llama-4-scout is).
+ return ChatGroq(
+ model=os.environ.get("GROQ_MODEL", "meta-llama/llama-4-scout-17b-16e-instruct"),
+ api_key=os.environ.get("GROQ_API_KEY", ""),
+ )
+ if _BACKEND == "claude":
+ from browser_use import ChatAnthropic
+ return ChatAnthropic(
+ model=os.environ.get("ANTHROPIC_MODEL", "claude-sonnet-4-6"),
+ api_key=os.environ.get("ANTHROPIC_API_KEY", ""),
+ max_tokens=4096,
+ timeout=120.0,
+ )
+ from browser_use import ChatOllama
+ return ChatOllama(model=_OLLAMA_MODEL, host=_OLLAMA_HOST)
-async def _screenshot_b64(page: Page) -> str:
+async def _screenshot_b64(browser_session) -> str:
+ """JPEG of the agent-focused tab, base64-encoded ('' if the browser isn't available)."""
try:
- jpeg = await page.screenshot(full_page=False, type="jpeg", quality=60)
+ jpeg = await browser_session.take_screenshot(format="jpeg", quality=60)
return base64.b64encode(jpeg).decode()
except Exception:
return ""
+def _has_page(browser_session) -> bool:
+ return bool(getattr(browser_session, "agent_focus_target_id", None))
+
+
def _push_frame(frame_queue: asyncio.Queue, frame_b64: str):
"""Drop oldest frame if full, then push newest."""
if frame_queue.full():
@@ -152,93 +102,130 @@ def _push_frame(frame_queue: asyncio.Queue, frame_b64: str):
pass
-async def _cdp_screenshotter(browser_session, frame_queue: asyncio.Queue):
+async def _cdp_screencaster(browser_session, frame_queue: asyncio.Queue):
"""
- Attach a CDP Page.startScreencast session to the active Playwright page.
- Chrome pushes JPEG frames at up to ~30fps without us having to poll.
- Re-attaches automatically when the agent navigates to a new tab.
+ Stream CDP Page.screencast frames from whichever tab currently has agent focus.
+ Chrome pushes JPEGs (no screenshot polling); we only poll cached session state to
+ notice focus changes and move the screencast to the new tab. Mirrors browser-use's own
+ RecordingWatchdog, which is inactive here because we don't enable video recording.
"""
- active: dict = {} # {"page": Page, "client": CDPSession}
-
- async def _attach(page):
- # Tear down previous session
- old_client = active.get("client")
- if old_client:
- try:
- await old_client.send("Page.stopScreencast")
- except Exception:
- pass
- try:
- await old_client.detach()
- except Exception:
- pass
- active.clear()
+ current: dict = {"session_id": None}
+ registered = False
+ ack_tasks: set[asyncio.Task] = set()
+ async def _ack(frame_session_id: int, session_id: str | None):
try:
- client = await page.context.new_cdp_session(page)
+ await browser_session.cdp_client.send.Page.screencastFrameAck(
+ params={"sessionId": frame_session_id}, session_id=session_id,
+ )
except Exception:
- return
+ pass
- active["page"] = page
- active["client"] = client
+ def on_frame(event, session_id):
+ # Synchronous: the CDP reader awaits handlers inline, so the ack runs as its own task.
+ if session_id == current["session_id"] and event.get("data"):
+ _push_frame(frame_queue, event["data"])
+ task = asyncio.create_task(_ack(event["sessionId"], session_id))
+ ack_tasks.add(task)
+ task.add_done_callback(ack_tasks.discard)
- async def on_frame(data):
- frame_b64 = data.get("data", "")
- if frame_b64:
- _push_frame(frame_queue, frame_b64)
- # Ack is required — Chrome stops sending frames without it
+ try:
+ while True:
try:
- await client.send("Page.screencastFrameAck", {"sessionId": data["sessionId"]})
+ if _has_page(browser_session):
+ if not registered:
+ browser_session.cdp_client.register.Page.screencastFrame(on_frame)
+ registered = True
+ cdp_session = await browser_session.get_or_create_cdp_session(focus=False)
+ if cdp_session.session_id != current["session_id"]:
+ old = current["session_id"]
+ current["session_id"] = cdp_session.session_id
+ if old:
+ try:
+ await browser_session.cdp_client.send.Page.stopScreencast(session_id=old)
+ except Exception:
+ pass # the old tab may already be gone
+ await cdp_session.cdp_client.send.Page.startScreencast(
+ params={"format": "jpeg", "quality": 60,
+ "maxWidth": VIEWPORT["width"], "maxHeight": VIEWPORT["height"],
+ "everyNthFrame": 1},
+ session_id=cdp_session.session_id,
+ )
+ except Exception as exc:
+ logger.debug("screencast attach failed: %s", exc)
+ current["session_id"] = None # retry on the next tick
+ await asyncio.sleep(0.5)
+ finally:
+ if current["session_id"]:
+ try:
+ await browser_session.cdp_client.send.Page.stopScreencast(session_id=current["session_id"])
except Exception:
pass
- client.on("Page.screencastFrame", on_frame)
- try:
- await client.send("Page.startScreencast", {
- "format": "jpeg",
- "quality": 60,
- "maxWidth": 1280,
- "maxHeight": 800,
- "everyNthFrame": 1,
- })
- except Exception:
- pass
-
- last_page = None
- while True:
- try:
- context = getattr(browser_session, "context", None)
- if context and context.pages:
- page = context.pages[-1]
- if page is not last_page:
- last_page = page
- await _attach(page)
- except Exception:
- pass
- await asyncio.sleep(0.3) # Only need to poll for page-change events
-
DONE_COMMANDS = {"submit", "done", "skip", "cancel", "abort"}
-
-
-async def _interactive_session(page: Page, instruction_queue: asyncio.Queue, context=None) -> AsyncGenerator[dict, None]:
+# Subset of DONE_COMMANDS that mean "the application went in" (vs. abandoned).
+SUBMIT_COMMANDS = {"submit", "done"}
+
+
+async def _focused_page(browser_session):
+ """Actor Page for the agent-focused tab, reusing its existing CDP session."""
+ cdp_session = await browser_session.get_or_create_cdp_session(focus=False)
+ return _ActorPage(browser_session, cdp_session.target_id, session_id=cdp_session.session_id), cdp_session
+
+
+async def _run_user_command(browser_session, cmd: dict) -> None:
+ """Replay one takeover input (from BrowserView.tsx) onto the focused tab via CDP."""
+ page, cdp_session = await _focused_page(browser_session)
+ kind = cmd.get("type", "")
+ if kind == "click":
+ tabs_before = len(browser_session.get_page_targets())
+ await (await page.mouse).click(int(float(cmd["x"])), int(float(cmd["y"])))
+ await asyncio.sleep(0.8)
+ # Follow a tab the click opened (e.g. "Apply on company site").
+ if len(browser_session.get_page_targets()) > tabs_before:
+ new_tab = await browser_session.get_most_recently_opened_target_id()
+ await browser_session.get_or_create_cdp_session(new_tab, focus=True)
+ elif kind == "type":
+ await _insert_text(browser_session, cdp_session, str(cmd.get("text", "")))
+ elif kind == "key":
+ await page.press(str(cmd.get("key", "Enter")))
+ await asyncio.sleep(0.4)
+ elif kind == "scroll":
+ await (await page.mouse).scroll(delta_y=int(float(cmd.get("delta", 300))))
+ await asyncio.sleep(0.3)
+
+
+async def _insert_text(browser_session, cdp_session, text: str) -> None:
+ await browser_session.cdp_client.send.Input.insertText(params={"text": text}, session_id=cdp_session.session_id)
+ await asyncio.sleep(0.2)
+
+
+async def _interactive_session(
+ browser_session,
+ instruction_queue: asyncio.Queue,
+ intro_action: str,
+ intro_reason: str,
+ on_submit: Callable[[], Awaitable[tuple[bool, str]]],
+) -> AsyncGenerator[dict, None]:
"""
- Hand control to the user. Loops processing commands from instruction_queue.
- Automatically follows new tabs opened by clicks.
+ Hand control to the user until they finish. Commands (from BrowserView.tsx):
+ JSON {type: click|type|key|scroll} → replayed onto the page via CDP
+ 'submit' → a short agent clicks the final submit button (submitted if it confirms)
+ 'done' → the user submitted it themselves
+ 'skip' / 'cancel' / 'abort' → abandon (not submitted)
+ any other text → typed into the focused field
"""
- screenshot = await _screenshot_b64(page)
- yield {
- "action": "You have control — click the screenshot or type below",
- "screenshot": screenshot or None,
- "blocked": True,
- "reason": "Click anywhere on the screenshot to interact, type in the box, or type 'submit' / 'skip'.",
- "done": False,
- "interactive": True,
- }
+ def interactive(action: str, reason: str, screenshot: str) -> dict:
+ return {"action": action, "screenshot": screenshot or None, "blocked": True,
+ "reason": reason, "done": False, "interactive": True}
+
+ yield interactive(intro_action, intro_reason, await _screenshot_b64(browser_session))
+ again = "Click the screenshot or type below. 'submit' sends it, 'done' if you submitted it yourself, 'skip' cancels."
while True:
try:
- raw = await asyncio.wait_for(instruction_queue.get(), timeout=120)
+ raw = await asyncio.wait_for(instruction_queue.get(), timeout=_INPUT_TIMEOUT_S)
except asyncio.TimeoutError:
yield {"action": "Session timed out", "screenshot": None, "blocked": False, "reason": None, "done": True}
return
@@ -250,69 +237,54 @@ async def _interactive_session(page: Page, instruction_queue: asyncio.Queue, con
cmd = parsed
except (json.JSONDecodeError, TypeError):
pass
-
- if cmd:
- kind = cmd.get("type", "")
- if kind == "click":
- pages_before = list(context.pages) if context else []
- await page.mouse.click(float(cmd["x"]), float(cmd["y"]))
- await page.wait_for_timeout(800)
- if context and len(context.pages) > len(pages_before):
- new_page = context.pages[-1]
- try:
- await new_page.wait_for_load_state("domcontentloaded", timeout=10000)
- page = new_page
- except Exception:
- pass
- elif kind == "type":
- await page.keyboard.type(str(cmd.get("text", "")))
- await page.wait_for_timeout(200)
- elif kind == "key":
- await page.keyboard.press(str(cmd.get("key", "Enter")))
- await page.wait_for_timeout(400)
- elif kind == "scroll":
- await page.mouse.wheel(0, float(cmd.get("delta", 300)))
- await page.wait_for_timeout(300)
- elif kind in DONE_COMMANDS or cmd.get("type") in DONE_COMMANDS:
- yield {"action": f"User ended session: {kind}", "screenshot": None, "blocked": False, "reason": None, "done": True}
+ word = (cmd.get("type", "") if cmd else str(raw)).strip().lower()
+
+ if word == "submit":
+ yield {"action": "Submitting the application…", "screenshot": None, "blocked": False,
+ "reason": None, "done": False}
+ ok, detail = await on_submit()
+ if ok:
+ yield {"action": f"Application submitted{': ' + detail if detail else ''}",
+ "screenshot": await _screenshot_b64(browser_session) or None,
+ "blocked": False, "reason": None, "done": True, "submitted": True}
return
+ yield interactive(
+ f"Couldn't confirm the submission{': ' + detail if detail else ''}",
+ "Click the submit button on the screenshot yourself, then type 'done' — or 'skip' to cancel.",
+ await _screenshot_b64(browser_session),
+ )
+ continue
+
+ if word in DONE_COMMANDS:
+ if word in SUBMIT_COMMANDS:
+ yield {"action": "Marked as submitted by you", "screenshot": None, "blocked": False,
+ "reason": None, "done": True, "submitted": True}
+ else:
+ yield {"action": "Cancelled by user", "screenshot": None, "blocked": False, "reason": None, "done": True}
+ return
- screenshot = await _screenshot_b64(page)
- yield {
- "action": f"Executed: {kind}",
- "screenshot": screenshot or None,
- "blocked": True,
- "reason": "Click the screenshot or type below. Type 'submit' when done.",
- "done": False,
- "interactive": True,
- }
-
- else:
- text = str(raw).strip().lower()
- if text in DONE_COMMANDS:
- if text in ("skip", "cancel", "abort"):
- yield {"action": "Cancelled by user", "screenshot": None, "blocked": False, "reason": None, "done": True}
- else:
- yield {"action": "User confirmed — proceeding", "screenshot": None, "blocked": False, "reason": None, "done": True}
- return
- await page.keyboard.type(str(raw))
- await page.wait_for_timeout(200)
- screenshot = await _screenshot_b64(page)
- yield {
- "action": f"Typed: {raw[:40]}",
- "screenshot": screenshot or None,
- "blocked": True,
- "reason": "Click the screenshot or type below. Type 'submit' when done.",
- "done": False,
- "interactive": True,
- }
+ try:
+ if cmd:
+ await _run_user_command(browser_session, cmd)
+ label = f"Executed: {word}"
+ else:
+ _, cdp_session = await _focused_page(browser_session)
+ await _insert_text(browser_session, cdp_session, str(raw))
+ label = f"Typed: {str(raw)[:40]}"
+ except Exception as exc:
+ logger.warning("takeover command %r failed: %s", word, exc)
+ label = f"That didn't work ({exc}) — try again"
+ yield interactive(label, again, await _screenshot_b64(browser_session))
def _build_task(job_url: str, persona: dict, cover_letter: str,
job_email: str, job_password: str, cv_path: str) -> str:
lines = []
if job_email:
- lines.append(f"If asked to log in, use: email={job_email}, password={job_password}")
+ # The real password is never in the prompt: browser-use substitutes the placeholder at
+ # input time (Agent sensitive_data, scoped to the job site — see _credential_scope).
+ password_hint = " password=job_password" if job_password else ""
+ lines.append(f"If asked to log in, use: email={job_email}{password_hint}")
if cv_path:
lines.append(f"If asked to upload a CV/resume, upload the file at: {cv_path}")
@@ -451,10 +423,76 @@ def _build_task(job_url: str, persona: dict, cover_letter: str,
task += f"\n\nCustom Directives to follow: {persona['custom_directives']}"
task += f"\n\nFull cover letter to paste into any cover letter field:\n{cover_letter}"
- task += "\n\nComplete and submit the application. Fill every required field. If a field is optional and you don't have the answer, leave it blank."
+ task += (
+ "\n\nFill in the application completely: every required field, plus optional ones you have an answer for "
+ "(leave optional fields blank if you don't). Logging in, creating an account, uploading the CV and moving "
+ "between steps of a multi-page form with 'Next'/'Continue' are all fine."
+ "\n\nIMPORTANT — DO NOT SUBMIT. Never click the FINAL button that sends the application "
+ "(e.g. 'Submit', 'Submit application', 'Apply', 'Send application', 'Finish'). The candidate reviews the "
+ "filled form and submits it themselves. As soon as the only thing left is that final submission, stop and "
+ "call done with success=true, summarising what you filled and anything you were unsure about. If you cannot "
+ "fill the form (blocked, missing information, page errors), call done with success=false and say why."
+ )
return task
+_SUBMIT_TASK = (
+ "The job application form on the current page has already been filled in and the candidate has reviewed "
+ "and approved it. Click the button that submits the application (e.g. 'Submit', 'Submit application', "
+ "'Apply', 'Send'). Do NOT change any field and do NOT navigate elsewhere. After clicking, check the page: "
+ "call done with success=true only if it confirms the application was received/submitted; otherwise call done "
+ "with success=false and quote any error or validation message shown."
+)
+
+
+def _browser_profile() -> "BrowserProfile":
+ return BrowserProfile(
+ headless=True,
+ keep_alive=True, # keep the browser after agent.run() for review / takeover / submit
+ viewport=VIEWPORT,
+ device_scale_factor=1,
+ block_ip_addresses=_BLOCK_IP_ADDRESSES,
+ prohibited_domains=_PROHIBITED_HOSTS,
+ enable_default_extensions=False,
+ args=[
+ "--no-sandbox",
+ "--disable-dev-shm-usage",
+ "--disable-blink-features=AutomationControlled",
+ "--disable-extensions",
+ # Hardware / performance
+ "--disable-gpu",
+ "--js-flags=--max-old-space-size=512",
+ ],
+ )
+
+
+def _credential_scope(job_url: str) -> str | None:
+ """
+ browser-use domain pattern the job-site password may be typed into: the job URL's host and
+ its subdomains, HTTPS only. A login on a different domain (e.g. an external ATS) won't get
+ the secret — the agent hands over and the user types it — which is the point: a
+ prompt-injected page elsewhere can't get it typed into an attacker's form.
+ """
+ from urllib.parse import urlparse
+ host = (urlparse(job_url).hostname or "").lower()
+ host = host.removeprefix("www.")
+ return f"https://*.{host}" if host else None
+
+
+async def _submit_with_agent(llm, browser_session) -> tuple[bool, str]:
+ """Run a tiny follow-up agent on the same (kept-alive) browser to click the final submit."""
+ try:
+ agent = Agent(
+ task=_SUBMIT_TASK, llm=llm, browser_session=browser_session,
+ use_vision=False, use_judge=False, directly_open_url=False,
+ )
+ history = await agent.run(max_steps=_MAX_SUBMIT_STEPS)
+ return bool(history.is_successful()), (history.final_result() or "")[:200]
+ except Exception as exc:
+ logger.warning("submit agent failed: %s", exc)
+ return False, str(exc)[:200]
+
+
async def apply_with_browser(
job_url: str,
cv_profile: dict,
@@ -466,9 +504,11 @@ async def apply_with_browser(
cv_path: str = "",
) -> AsyncGenerator[dict, None]:
"""
- Async generator — uses browser-use LLM agent to autonomously complete a job application.
- Falls back to interactive session if the agent fails.
- Yields dicts: { action, screenshot (base64|None), blocked (bool), reason (str|None), done (bool) }
+ Async generator: the browser-use agent fills the application, then the user reviews it and
+ decides whether to submit (see module docstring). Yields dicts:
+ { action, screenshot (base64|None), blocked (bool), reason (str|None), done (bool),
+ interactive (bool, optional), submitted (bool, only on the final done event) }
+ The browser is always killed on exit — including when the client disconnects mid-run.
"""
if not BROWSER_USE_AVAILABLE:
yield {
@@ -481,32 +521,15 @@ async def apply_with_browser(
yield {"action": "Browser agent starting…", "screenshot": None, "blocked": False, "reason": None, "done": False}
- browser_profile = BrowserProfile(
- headless=True,
- disable_security=True,
- enable_default_extensions=False,
- args=[
- "--no-sandbox",
- "--disable-dev-shm-usage",
- "--disable-blink-features=AutomationControlled",
- "--window-size=1280,800",
- "--disable-extensions",
- # Hardware / performance
- "--disable-gpu",
- "--js-flags=--max-old-space-size=512",
- ],
- )
- browser_session = BrowserSession(browser_profile=browser_profile)
+ llm = _make_browser_llm()
+ browser_session = BrowserSession(browser_profile=_browser_profile())
step_queue: asyncio.Queue = asyncio.Queue()
async def on_step(browser_state, agent_output, step_number):
- screenshot = browser_state.screenshot # already base64 JPEG
+ screenshot = getattr(browser_state, "screenshot", None) # base64; None when use_vision=False
if frame_queue is not None and screenshot:
- try:
- frame_queue.put_nowait(screenshot)
- except asyncio.QueueFull:
- pass
+ _push_frame(frame_queue, screenshot)
goal = getattr(agent_output, "next_goal", "") or ""
actions = getattr(agent_output, "action", None) or []
action_str = str(goal or (actions[0] if actions else agent_output))[:120]
@@ -518,106 +541,86 @@ async def on_step(browser_state, agent_output, step_number):
"done": False,
})
- llm = _make_browser_llm()
+ scope = _credential_scope(job_url)
+ sensitive_data = {scope: {"job_password": job_password}} if (job_password and scope) else None
- agent = Agent(
- task=task,
- llm=llm,
- browser_session=browser_session,
- register_new_step_callback=on_step,
- use_vision=False,
- available_file_paths=[cv_path] if cv_path else [],
- )
- agent_task = asyncio.create_task(agent.run())
- screenshotter_task = (
- asyncio.create_task(_cdp_screenshotter(browser_session, frame_queue))
- if frame_queue is not None else None
- )
+ agent_task: asyncio.Task | None = None
+ screencast_task: asyncio.Task | None = None
+ try:
+ agent = Agent(
+ task=task,
+ llm=llm,
+ browser_session=browser_session,
+ register_new_step_callback=on_step,
+ use_vision=False,
+ available_file_paths=[cv_path] if cv_path else [],
+ sensitive_data=sensitive_data,
+ # The judge LLM call grades the run from the RAW step history, which browser-use does not
+ # redact — e.g. an evaluate() that reads the form back carries the typed password. We
+ # rely on the agent's own is_successful(), so the judge only costs a call and leaks.
+ use_judge=False,
+ # Open the job page before step 1. browser-use's own directly_open_url guesses the URL
+ # from the task text and gives up when it sees more than one URL-like string — which a
+ # persona's LinkedIn/GitHub links or a cover letter mentioning "ASP.NET" always trigger.
+ initial_actions=[{"navigate": {"url": job_url, "new_tab": False}}],
+ )
+ agent_task = asyncio.create_task(agent.run(max_steps=_MAX_FILL_STEPS))
+ if frame_queue is not None:
+ screencast_task = asyncio.create_task(_cdp_screencaster(browser_session, frame_queue))
- # Stream step events while agent runs
- while not agent_task.done():
- try:
- event = await asyncio.wait_for(step_queue.get(), timeout=1.0)
- yield event
- except asyncio.TimeoutError:
- pass
+ # Stream step events while the agent fills the form
+ while not agent_task.done():
+ try:
+ yield await asyncio.wait_for(step_queue.get(), timeout=1.0)
+ except asyncio.TimeoutError:
+ pass
+ while not step_queue.empty():
+ yield step_queue.get_nowait()
- # Stop continuous screenshotter
- if screenshotter_task:
- screenshotter_task.cancel()
+ agent_error: Exception | None = None
+ ready = False
+ summary = ""
try:
- await screenshotter_task
- except asyncio.CancelledError:
- pass
-
- # Drain any remaining buffered events
- while not step_queue.empty():
- yield await step_queue.get()
+ history = agent_task.result()
+ ready = bool(history.is_successful()) # agent's own verdict: "form filled, ready to submit"
+ summary = (history.final_result() or "")[:300]
+ except Exception as exc:
+ agent_error = exc
+ logger.warning("browser agent failed: %s", exc)
+
+ if not _has_page(browser_session):
+ yield {"action": f"Browser error: {agent_error or 'no page was opened'}", "screenshot": None,
+ "blocked": False, "reason": None, "done": True}
+ return
- # Check agent result
- agent_error = None
- try:
- result = agent_task.result()
- result_str = str(result)[:200] if result else "Done"
- yield {"action": f"Agent finished: {result_str}", "screenshot": None, "blocked": False, "reason": None, "done": False}
- except Exception as e:
- agent_error = e
-
- # Get current Playwright page via browser_session.context
- page = None
- pw_context = None
- try:
- pw_context = getattr(browser_session, "context", None)
- if pw_context and pw_context.pages:
- page = pw_context.pages[-1]
- except Exception:
- pass
+ if agent_error:
+ intro = f"The agent hit a problem — handing control to you ({agent_error})"
+ reason = "Finish the form yourself, then type 'submit' (or 'done' if you submitted it), or 'skip'."
+ elif ready:
+ intro = "Form filled — review it before anything is sent" + (f". Agent notes: {summary}" if summary else "")
+ reason = "Type 'submit' to send it, click/type to fix anything first, or 'skip' to cancel."
+ else:
+ intro = "The agent couldn't finish the form" + (f": {summary}" if summary else "")
+ reason = "Take over by clicking/typing, then 'submit' (or 'done' if you submitted it), or 'skip'."
- if agent_error and page:
- screenshot = await _screenshot_b64(page)
- yield {
- "action": f"Agent encountered an issue — handing control to you ({agent_error})",
- "screenshot": screenshot or None,
- "blocked": True,
- "reason": "Take over to complete the application, then type 'submit'.",
- "done": False,
- "interactive": True,
- }
- async for step in _interactive_session(page, instruction_queue, pw_context):
+ async for step in _interactive_session(
+ browser_session, instruction_queue, intro, reason,
+ on_submit=lambda: _submit_with_agent(llm, browser_session),
+ ):
yield step
if step.get("done"):
+ return
+ finally:
+ # Runs on normal completion AND when the client disconnects (generator closed):
+ # stop background work and kill Chromium so no browser process is leaked.
+ for t in (agent_task, screencast_task):
+ if t and not t.done():
+ t.cancel()
try:
- await browser_session.close()
- except Exception:
+ await t
+ except BaseException:
pass
- return
- elif agent_error:
- yield {"action": f"Browser error: {agent_error}", "screenshot": None, "blocked": False, "reason": None, "done": True}
try:
- await browser_session.close()
- except Exception:
- pass
- return
-
- # Final interactive review before submit
- if page:
- screenshot = await _screenshot_b64(page)
- yield {
- "action": "Review the filled form — type 'submit' to submit or 'skip' to skip",
- "screenshot": screenshot or None,
- "blocked": True,
- "reason": "Check the form looks correct before submitting.",
- "done": False,
- "interactive": True,
- }
- async for step in _interactive_session(page, instruction_queue, pw_context):
- yield step
- if step.get("done"):
- break
- else:
- yield {"action": "Application complete", "screenshot": None, "blocked": False, "reason": None, "done": True}
-
- try:
- await browser_session.close()
- except Exception:
- pass
+ await browser_session.kill()
+ except Exception as exc:
+ logger.debug("browser kill failed: %s", exc)
diff --git a/api/app/crypto.py b/api/app/crypto.py
new file mode 100644
index 0000000..c3700f1
--- /dev/null
+++ b/api/app/crypto.py
@@ -0,0 +1,37 @@
+"""
+Encryption at rest for stored third-party credentials (the user's job-site password).
+
+Fernet (AES-128-CBC + HMAC-SHA256) keyed by CREDENTIALS_KEY. Kept separate from JWT_SECRET
+so rotating the login-token secret doesn't make stored credentials undecryptable.
+"""
+import logging
+import os
+
+from cryptography.fernet import Fernet, InvalidToken
+
+logger = logging.getLogger("windrush.crypto")
+
+_KEY = os.environ.get("CREDENTIALS_KEY", "")
+if not _KEY:
+ raise RuntimeError(
+ "CREDENTIALS_KEY is not set — generate one with: "
+ "python -c \"from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())\" "
+ "and add it to .env"
+ )
+try:
+ _fernet = Fernet(_KEY.encode())
+except (ValueError, TypeError) as exc:
+ raise RuntimeError("CREDENTIALS_KEY is not a valid Fernet key (32 url-safe base64-encoded bytes)") from exc
+
+
+def encrypt(plaintext: str) -> str:
+ return _fernet.encrypt(plaintext.encode("utf-8")).decode("ascii")
+
+
+def decrypt(token: str) -> str:
+ """Decrypt a stored token; '' if it can't be (e.g. CREDENTIALS_KEY was changed)."""
+ try:
+ return _fernet.decrypt(token.encode("ascii")).decode("utf-8")
+ except (InvalidToken, ValueError):
+ logger.warning("Stored credential could not be decrypted — was CREDENTIALS_KEY changed?")
+ return ""
diff --git a/api/app/discovery.py b/api/app/discovery.py
new file mode 100644
index 0000000..ad9f256
--- /dev/null
+++ b/api/app/discovery.py
@@ -0,0 +1,113 @@
+"""
+Background live job discovery for the feed.
+
+When the local jobs table has too few matches, /jobs used to run the whole live search
+(a Chromium scrape + ~70 ATS/API calls) and then embed every result *synchronously on the
+event loop* — stalling every other request for the duration. Now /jobs calls ensure(),
+which starts that work as a background task and returns immediately; the frontend re-polls
+the feed while `discovering` is true.
+"""
+import asyncio
+import logging
+import multiprocessing
+import time
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures.process import BrokenProcessPool
+
+from . import jobs_db
+
+logger = logging.getLogger("windrush.discovery")
+
+_COOLDOWN_S = 30 * 60 # don't re-run the same (query, location) within this window
+_MAX_CONCURRENT = 2 # each run may launch Chromium; cap total load
+_running: dict[tuple[str, str], asyncio.Task] = {}
+_finished_at: dict[tuple[str, str], float] = {}
+
+# The live search runs in worker *processes*: it fans out to ~70 job boards and does a lot of
+# CPU work on the results (multi-MB JSON, HTML-to-text, exposure scoring). In threads that
+# work contends for the GIL with the event loop and stalls the API for ~1s at a time; in a
+# separate process it can't. 'spawn' avoids forking a process that has live threads.
+_pool: ProcessPoolExecutor | None = None
+
+
+def _get_pool() -> ProcessPoolExecutor:
+ global _pool
+ if _pool is None:
+ _pool = ProcessPoolExecutor(max_workers=_MAX_CONCURRENT,
+ mp_context=multiprocessing.get_context("spawn"))
+ return _pool
+
+
+def shutdown() -> None:
+ """Stop worker processes (called from the app's lifespan on exit)."""
+ global _pool
+ if _pool is not None:
+ _pool.shutdown(wait=False, cancel_futures=True)
+ _pool = None
+
+
+def _search_in_process(query: str, location: str) -> list[dict]:
+ """Worker-process entry point: run the async multi-source search on its own event loop."""
+ from .job_proxy import search_jobs
+ return asyncio.run(search_jobs(query, location))
+
+
+async def search_jobs(query: str, location: str) -> list[dict]:
+ """Live multi-source search, executed in a worker process."""
+ global _pool
+ try:
+ return await asyncio.get_running_loop().run_in_executor(_get_pool(), _search_in_process, query, location)
+ except BrokenProcessPool:
+ _pool = None # a worker died (e.g. Chromium OOM) — start a fresh pool next time
+ raise
+
+
+def _key(query: str, location: str) -> tuple[str, str]:
+ return (" ".join(query.lower().split()), " ".join(location.lower().split()))
+
+
+def is_running(query: str, location: str) -> bool:
+ task = _running.get(_key(query, location))
+ return bool(task and not task.done())
+
+
+def ensure(query: str, location: str) -> bool:
+ """
+ Start a background discovery for (query, location) unless one is running or ran recently.
+ Returns True while a discovery for this key is in flight (the caller reports `discovering`).
+ """
+ key = _key(query, location)
+ if is_running(query, location):
+ return True
+ if time.monotonic() - _finished_at.get(key, -_COOLDOWN_S) < _COOLDOWN_S:
+ return False
+ task = asyncio.create_task(_discover(query, location), name=f"discover:{key}")
+ _running[key] = task
+ task.add_done_callback(lambda t, k=key: _done(k, t))
+ return True
+
+
+def _done(key: tuple[str, str], task: asyncio.Task) -> None:
+ _running.pop(key, None)
+ _finished_at[key] = time.monotonic()
+ if len(_finished_at) > 1000: # bound memory: forget the oldest entries
+ for k in sorted(_finished_at, key=_finished_at.get)[:500]:
+ _finished_at.pop(k, None)
+ if not task.cancelled() and task.exception():
+ logger.warning("discovery %r failed: %s", key, task.exception())
+
+
+async def _discover(query: str, location: str) -> None:
+ from .main import _infer_level # deferred: main imports this module
+ started = time.monotonic()
+ jobs = await search_jobs(query, location) # concurrency is capped by the pool size
+ # search_jobs falls back to the bundled mock fixture when every live source is empty;
+ # those must never be persisted as real listings.
+ live = [dict(j) for j in jobs if j.get("source") != "fixture"]
+ for j in live:
+ j["source"] = j.get("source") or "live"
+ j["level"] = _infer_level(j.get("title", ""))
+ # Embedding is mostly waiting on Ollama, so a thread is enough here.
+ added, refreshed = (await asyncio.to_thread(jobs_db.add_jobs, live)) if live else (0, 0)
+ logger.info("discovery %r/%r: %d found, %d added, %d refreshed in %.1fs",
+ query, location, len(live), added, refreshed, time.monotonic() - started)
diff --git a/api/app/guardrails.py b/api/app/guardrails.py
index 29c5e12..801b75e 100644
--- a/api/app/guardrails.py
+++ b/api/app/guardrails.py
@@ -1,18 +1,14 @@
"""
Windrush Guardrails Module
-Deterministic pre/post-execution checks for all agent tool calls.
+Deterministic prompt-injection screening for uploaded CVs, before their text reaches an LLM.
No external dependencies — pure stdlib.
"""
-import json
import logging
import re
-import time
-from dataclasses import dataclass, field
-from typing import Any
logger = logging.getLogger("windrush.guardrails")
-# ── 1. Prompt injection patterns (CV input check) ───────────────────────────
+# ── Prompt injection patterns (CV input check) ───────────────────────────────
CV_INJECTION_PATTERNS: list[re.Pattern] = [
re.compile(r"ignore\s+(previous|all|prior|above)\s+instructions", re.I),
@@ -30,66 +26,6 @@
re.compile(r"send\s+(my\s+)?(data|information|cv|profile)\s+to", re.I),
]
-# ── 2. Search query sanitisation patterns ────────────────────────────────────
-
-SEARCH_INJECTION_PATTERNS: list[re.Pattern] = [
- re.compile(r"ignore\s+instructions", re.I),
- re.compile(r"system\s*prompt", re.I),
- re.compile(r"<\s*\/?script", re.I),
- re.compile(r"[<>]{2,}"),
- re.compile(r"\bexec\s*\(", re.I),
- re.compile(r"\beval\s*\(", re.I),
- re.compile(r"\bdrop\s+table\b", re.I),
- re.compile(r"\bSELECT\s+\*\b", re.I),
-]
-
-# ── 3. PII redaction patterns (SSE output only) ──────────────────────────────
-
-PII_PATTERNS: list[tuple[re.Pattern, str]] = [
- (re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}"), "[email]"),
- # UK landline / mobile
- (re.compile(r"(\+?44[\s\-]?|0)[1-9][\d\s\-]{7,12}"), "[phone]"),
- # Generic international
- (re.compile(r"\+\d{1,3}[\s\-]?\d{6,14}"), "[phone]"),
- # UK postcodes (e.g. SW1A 2AA)
- (re.compile(r"\b[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}\b", re.I), "[postcode]"),
- # NI numbers
- (re.compile(r"\b[A-CEGHJ-PR-TW-Z]{2}\s?\d{2}\s?\d{2}\s?\d{2}\s?[A-D]\b", re.I), "[ni-number]"),
-]
-
-# ── 4. Credential field names ────────────────────────────────────────────────
-
-CREDENTIAL_FIELD_NAMES = {"password", "job_password", "secret", "token", "api_key"}
-
-# Tools whose results contain derived CV data and should have PII masked in SSE
-_PII_REDACT_TOOLS = {"extract_cv_profile", "generate_cover_letter", "score_job_fit", "apply_to_job", "generate_skill_roadmap"}
-
-# ── 5. Audit log ─────────────────────────────────────────────────────────────
-
-@dataclass
-class GuardrailEvent:
- timestamp: float
- tool_name: str
- check: str
- fired: bool
- detail: str = ""
-
-
-_audit_log: list[GuardrailEvent] = []
-
-
-def _record(event: GuardrailEvent) -> None:
- _audit_log.append(event)
- if event.fired:
- logger.warning(
- "GUARDRAIL fired | tool=%s check=%s detail=%s",
- event.tool_name, event.check, event.detail,
- )
- else:
- logger.debug("GUARDRAIL ok | tool=%s check=%s", event.tool_name, event.check)
-
-
-# ── 6. Public exceptions ─────────────────────────────────────────────────────
class GuardrailViolation(Exception):
"""Raised when a hard guardrail fires."""
@@ -99,152 +35,16 @@ def __init__(self, check: str, detail: str):
super().__init__(f"Guardrail [{check}]: {detail}")
-# ── 7. Public API ─────────────────────────────────────────────────────────────
-
def check_cv_for_injection(cv_text: str) -> None:
"""
Scan raw CV text for prompt injection before it is sent to the LLM.
- Raises GuardrailViolation if a pattern matches.
- Call site: main.py, before run_pipeline().
+ Raises GuardrailViolation if a pattern matches. Call site: main.py /upload.
"""
for pattern in CV_INJECTION_PATTERNS:
m = pattern.search(cv_text)
if m:
- _record(GuardrailEvent(
- timestamp=time.time(),
- tool_name="cv_upload",
- check="cv_injection",
- fired=True,
- detail=f"Pattern '{pattern.pattern}' matched at pos {m.start()}",
- ))
+ logger.warning("GUARDRAIL fired | check=cv_injection pattern=%r pos=%d", pattern.pattern, m.start())
raise GuardrailViolation(
"cv_injection",
"CV text contains a potential prompt injection attempt and was rejected.",
)
- _record(GuardrailEvent(time.time(), "cv_upload", "cv_injection", False))
-
-
-def sanitise_tool_input(tool_name: str, tool_input: dict) -> dict:
- """
- Validate and sanitise a tool's input dict before execution.
- Returns a (possibly modified) copy. Raises GuardrailViolation for hard blocks.
- """
- cleaned = dict(tool_input)
-
- if tool_name in ("search_jobs", "web_search"):
- query = cleaned.get("query", "")
- for pattern in SEARCH_INJECTION_PATTERNS:
- if pattern.search(query):
- _record(GuardrailEvent(
- time.time(), tool_name, "search_injection", True,
- detail=f"Pattern '{pattern.pattern}' in query",
- ))
- raise GuardrailViolation(
- "search_injection",
- "Search query contains a disallowed pattern and was blocked.",
- )
- # Strip any residual HTML tags
- cleaned["query"] = re.sub(r"<[^>]+>", "", query).strip()
- _record(GuardrailEvent(time.time(), tool_name, "search_sanitise", False))
-
- if tool_name == "apply_to_job":
- job_url = cleaned.get("job_url", "")
- if job_url and not re.match(r"^https?://", job_url, re.I):
- _record(GuardrailEvent(
- time.time(), tool_name, "url_validation", True,
- detail=f"Non-HTTP URL rejected: {job_url[:60]}",
- ))
- raise GuardrailViolation("url_validation", "Job URL must begin with http:// or https://")
- _record(GuardrailEvent(time.time(), tool_name, "url_validation", False))
-
- # Strip credential fields from any tool input (defensive belt-and-braces)
- for field_name in list(cleaned.keys()):
- if field_name in CREDENTIAL_FIELD_NAMES:
- _record(GuardrailEvent(
- time.time(), tool_name, "credential_strip", True,
- detail=f"Stripped credential field '{field_name}' from {tool_name} input",
- ))
- del cleaned[field_name]
-
- return cleaned
-
-
-def _redact_value(v: Any) -> Any:
- """Recursively walk a Python object and apply PII regex patterns to strings only."""
- if isinstance(v, str):
- for pattern, replacement in PII_PATTERNS:
- v = pattern.sub(replacement, v)
- return v
- elif isinstance(v, dict):
- return {k: _redact_value(val) for k, val in v.items()}
- elif isinstance(v, list):
- return [_redact_value(item) for item in v]
- return v
-
-
-def redact_pii_from_result(tool_name: str, result: Any) -> tuple[Any, bool]:
- """
- Walk the result recursively and apply PII redaction patterns to string values.
- Returns (redacted_result, did_fire).
-
- IMPORTANT: Only call this for the SSE-streamed copy.
- The internal result passed to messages[] must remain unredacted.
- """
- if tool_name not in _PII_REDACT_TOOLS:
- return result, False
-
- redacted = _redact_value(result)
- fired = redacted != result
- _record(GuardrailEvent(
- time.time(), tool_name, "pii_redact", fired,
- detail="PII found and redacted" if fired else "",
- ))
- return redacted, fired
-
-
-def redact_pii_from_input(tool_name: str, tool_input: dict) -> tuple[dict, bool]:
- """
- Walk tool_input recursively and apply PII redaction patterns to string values.
- Returns (redacted_input, did_fire). Safe for SSE streaming only — never pass
- the redacted copy to execute_tool().
- """
- redacted = _redact_value(tool_input)
- fired = redacted != tool_input
- _record(GuardrailEvent(
- time.time(), tool_name, "pii_redact_input", fired,
- detail="PII redacted from tool input display" if fired else "",
- ))
- return redacted, fired
-
-
-def redact_credentials_from_input(tool_name: str, tool_input: dict) -> tuple[dict, bool]:
- """
- Return a copy of tool_input with credential values replaced by '***'.
- Safe for SSE streaming. The original tool_input is still used for execution.
- """
- cleaned = dict(tool_input)
- fired = False
- for field_name in CREDENTIAL_FIELD_NAMES:
- if field_name in cleaned:
- cleaned[field_name] = "***"
- fired = True
- if fired:
- _record(GuardrailEvent(
- time.time(), tool_name, "credential_sse_mask", True,
- detail=f"Masked credentials in SSE tool_call event for {tool_name}",
- ))
- return cleaned, fired
-
-
-def get_audit_log() -> list[dict]:
- """Return the in-process audit log as a list of dicts."""
- return [
- {
- "timestamp": e.timestamp,
- "tool_name": e.tool_name,
- "check": e.check,
- "fired": e.fired,
- "detail": e.detail,
- }
- for e in _audit_log
- ]
diff --git a/api/app/http_client.py b/api/app/http_client.py
new file mode 100644
index 0000000..3085807
--- /dev/null
+++ b/api/app/http_client.py
@@ -0,0 +1,22 @@
+"""
+httpx client factories that share one TLS context.
+
+httpx builds a fresh SSLContext for every Client/AsyncClient, which loads the whole CA bundle
+synchronously (~0.5–1s). Inside async code that stalls the event loop for every request in
+the process — and live job discovery alone creates ~70 clients. Building the context once,
+at import (i.e. startup), makes client construction ~0.1ms. SSLContext is safe to share.
+"""
+import ssl
+
+import certifi
+import httpx
+
+SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
+
+
+def async_client(**kwargs) -> httpx.AsyncClient:
+ return httpx.AsyncClient(verify=SSL_CONTEXT, **kwargs)
+
+
+def client(**kwargs) -> httpx.Client:
+ return httpx.Client(verify=SSL_CONTEXT, **kwargs)
diff --git a/api/app/job_searcher.py b/api/app/job_searcher.py
index 1fd21ac..623d0eb 100644
--- a/api/app/job_searcher.py
+++ b/api/app/job_searcher.py
@@ -20,6 +20,8 @@
import httpx
+from . import http_client
+from . import net_guard
from .risk_scorer import occupation_exposure
logger = logging.getLogger("windrush.job_searcher")
@@ -53,6 +55,7 @@
def _load_fixture() -> list[dict]:
jobs = json.loads(_FIXTURE_PATH.read_text())
for job in jobs:
+ job["source"] = "fixture" # mock data: shown as a fallback, never persisted (see discovery.py)
if "exposure_score" not in job:
job["exposure_score"] = round(occupation_exposure(job["title"]), 3)
return jobs
@@ -109,8 +112,8 @@ def _html_to_text(raw: str) -> str:
return text.strip()
-def _extract_jsonld_jobposting(raw_html: str) -> str:
- """Return the description from a JSON-LD JobPosting block, if present."""
+def _find_jsonld_jobposting(raw_html: str) -> dict:
+ """Return the first JSON-LD JobPosting object on the page, or {} if none."""
for m in re.finditer(
r'(?is)]+type=["\']application/ld\+json["\'][^>]*>(.*?)',
raw_html,
@@ -123,10 +126,15 @@ def _extract_jsonld_jobposting(raw_html: str) -> str:
t = obj.get("@type", "")
types = t if isinstance(t, list) else [t]
if any("JobPosting" in str(x) for x in types):
- desc = obj.get("description", "")
- if desc:
- return _html_to_text(desc)
- return ""
+ return obj
+ return {}
+
+
+def _extract_jsonld_jobposting(raw_html: str) -> str:
+ """Return the description from a JSON-LD JobPosting block, if present."""
+ obj = _find_jsonld_jobposting(raw_html)
+ desc = obj.get("description", "") if obj else ""
+ return _html_to_text(desc) if desc else ""
_BROWSER_HEADERS = {
@@ -149,7 +157,7 @@ async def _ats_full_description(url: str) -> str:
host = parsed.netloc.lower()
path = parsed.path
- async with httpx.AsyncClient(follow_redirects=True, timeout=12.0) as client:
+ async with http_client.async_client(follow_redirects=True, timeout=12.0) as client:
# Greenhouse — boards(-api).greenhouse.io/{slug}/jobs/{id}
if "greenhouse.io" in host:
m = re.search(r"/([^/]+)/jobs/(\d+)", path)
@@ -197,10 +205,10 @@ async def fetch_full_description(url: str, max_chars: int = 8000) -> str:
logger.debug("ATS description fetch failed for %s: %s", url[:80], exc)
try:
- async with httpx.AsyncClient(
- headers=_BROWSER_HEADERS, follow_redirects=True, timeout=15.0,
+ async with http_client.async_client(
+ headers=_BROWSER_HEADERS, timeout=15.0,
) as client:
- resp = await client.get(url)
+ resp = await net_guard.safe_get(client, url) # user-influenced URL: no internal hosts
resp.raise_for_status()
raw_html = resp.text
jd = _extract_jsonld_jobposting(raw_html)
@@ -212,6 +220,152 @@ async def fetch_full_description(url: str, max_chars: int = 8000) -> str:
return ""
+# ── User-pasted-URL scraping (build a full Job from an arbitrary listing) ──────
+
+def _meta_content(raw_html: str, prop: str) -> str:
+ """Return the content of a tag, if present."""
+ for attr in ("property", "name"):
+ m = re.search(
+ rf'(?is)]+{attr}=["\']{re.escape(prop)}["\'][^>]*>',
+ raw_html,
+ )
+ if m:
+ c = re.search(r'(?is)content=["\'](.*?)["\']', m.group(0))
+ if c:
+ return _html.unescape(c.group(1)).strip()
+ return ""
+
+
+def _first_tag_text(raw_html: str, tag: str) -> str:
+ m = re.search(rf"(?is)<{tag}[^>]*>(.*?)", raw_html)
+ return _html_to_text(m.group(1)) if m else ""
+
+
+def _jsonld_org_name(obj: dict) -> str:
+ org = obj.get("hiringOrganization")
+ if isinstance(org, dict):
+ return str(org.get("name", "")).strip()
+ if isinstance(org, str):
+ return org.strip()
+ return ""
+
+
+def _jsonld_location(obj: dict) -> str:
+ loc = obj.get("jobLocation")
+ if isinstance(loc, list):
+ loc = loc[0] if loc else {}
+ if isinstance(loc, dict):
+ addr = loc.get("address", loc)
+ if isinstance(addr, dict):
+ parts = [
+ addr.get("addressLocality", ""),
+ addr.get("addressRegion", ""),
+ addr.get("addressCountry", "")
+ if isinstance(addr.get("addressCountry"), str)
+ else (addr.get("addressCountry") or {}).get("name", ""),
+ ]
+ return ", ".join(p for p in parts if p)
+ if obj.get("jobLocationType") == "TELECOMMUTE":
+ return "Remote"
+ return ""
+
+
+def _jsonld_salary(obj: dict) -> tuple[float | None, float | None]:
+ base = obj.get("baseSalary")
+ if not isinstance(base, dict):
+ return None, None
+ value = base.get("value", base)
+ if not isinstance(value, dict):
+ return None, None
+
+ def _num(x):
+ try:
+ return float(x)
+ except (TypeError, ValueError):
+ return None
+
+ lo = _num(value.get("minValue")) or _num(value.get("value"))
+ hi = _num(value.get("maxValue")) or lo
+ return lo, hi
+
+
+async def scrape_job_from_url(url: str) -> dict:
+ """
+ Scrape an arbitrary job-posting URL into a full Job dict ready for analysis.
+
+ Reuses the ATS/JSON-LD description fetcher and adds structured-metadata
+ extraction (title, company, location, salary) from the page's JSON-LD
+ JobPosting block, falling back to OpenGraph / / <h1> tags. Raises
+ ValueError if the page yields neither a usable title nor a description, so
+ the caller can surface a clear "couldn't read that listing" message.
+ """
+ import uuid
+ from urllib.parse import urlparse
+
+ if not url or not url.startswith("http"):
+ raise ValueError("Please paste a valid http(s) job-posting link.")
+ # Fail loudly (400) rather than as a generic "couldn't read" — UnsafeURLError is a ValueError.
+ await net_guard.assert_public_url(url)
+
+ description = await fetch_full_description(url)
+
+ raw_html = ""
+ try:
+ async with http_client.async_client(
+ headers=_BROWSER_HEADERS, timeout=15.0,
+ ) as client:
+ resp = await net_guard.safe_get(client, url) # user-influenced URL: no internal hosts
+ resp.raise_for_status()
+ raw_html = resp.text
+ except Exception as exc:
+ logger.debug("Page fetch failed for %s: %s", url[:80], exc)
+
+ posting = _find_jsonld_jobposting(raw_html) if raw_html else {}
+
+ title = str(posting.get("title", "")).strip()
+ company = _jsonld_org_name(posting)
+ location = _jsonld_location(posting)
+ salary_min, salary_max = _jsonld_salary(posting)
+
+ # Description fallback: JSON-LD object may carry it even when the ATS/standalone
+ # fetcher returned "" (e.g. it was < 200 chars there).
+ if not description and posting.get("description"):
+ description = _html_to_text(posting["description"])[:_DESC_MAX].strip()
+
+ # OpenGraph / HTML fallbacks for missing metadata.
+ if not title:
+ title = _meta_content(raw_html, "og:title") or _first_tag_text(raw_html, "h1")
+ if not title:
+ page_title = _first_tag_text(raw_html, "title")
+ title = re.split(r"\s+[|\-–—]\s+", page_title)[0].strip() if page_title else ""
+ if not company:
+ company = _meta_content(raw_html, "og:site_name")
+ if not company:
+ company = urlparse(url).netloc.replace("www.", "")
+
+ title = title.strip()
+ if not title and not description:
+ raise ValueError(
+ "Couldn't read that listing — try a direct job-posting link "
+ "(e.g. a Greenhouse, Lever, or a page with a standard job schema)."
+ )
+
+ job: dict = {
+ "id": uuid.uuid4().hex,
+ "job_id": uuid.uuid4().hex[:12],
+ "title": title or "Job posting",
+ "company": company or "Unknown",
+ "location": location,
+ "description": description,
+ "url": url,
+ "salary_min": salary_min,
+ "salary_max": salary_max,
+ "source": "user-url",
+ }
+ job["exposure_score"] = round(occupation_exposure(job["title"]), 3)
+ return job
+
+
# ── Title filter (from career-ops/portals.yml) ────────────────────────────────
_POSITIVE_KW: list[str] = [
@@ -556,36 +710,42 @@ async def _fetch_greenhouse(slug: str, company: str, keywords: list[str]) -> lis
try:
# content=true is required — without it the API omits the JD entirely.
url = f"https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true"
- async with httpx.AsyncClient(timeout=8.0) as client:
+ async with http_client.async_client(timeout=8.0) as client:
resp = await client.get(url)
resp.raise_for_status()
- data = resp.json()
- jobs = []
- for item in data.get("jobs", []):
- title = item.get("title", "")
- if not _title_matches_query(title, keywords):
- continue
- location = item.get("location", {}).get("name", "")
- jobs.append({
- "job_id": f"gh-{item.get('id', '')}",
- "title": title,
- "company": company,
- "location": location or "Remote",
- "description": _html_to_text(item.get("content") or "")[:_DESC_MAX],
- "url": item.get("absolute_url", ""),
- "salary_min": None,
- "salary_max": None,
- "exposure_score": round(occupation_exposure(title), 3),
- })
- return jobs
+ # Big boards return every posting's full HTML (several MB); decoding + HTML-to-text +
+ # exposure scoring is ~1s of CPU, so do it off the event loop.
+ return await asyncio.to_thread(_greenhouse_jobs, resp.content, company, keywords)
except Exception:
return []
+def _greenhouse_jobs(raw: bytes, company: str, keywords: list[str]) -> list[dict]:
+ data = json.loads(raw)
+ jobs = []
+ for item in data.get("jobs", []):
+ title = item.get("title", "")
+ if not _title_matches_query(title, keywords):
+ continue
+ location = item.get("location", {}).get("name", "")
+ jobs.append({
+ "job_id": f"gh-{item.get('id', '')}",
+ "title": title,
+ "company": company,
+ "location": location or "Remote",
+ "description": _html_to_text(item.get("content") or "")[:_DESC_MAX],
+ "url": item.get("absolute_url", ""),
+ "salary_min": None,
+ "salary_max": None,
+ "exposure_score": round(occupation_exposure(title), 3),
+ })
+ return jobs
+
+
async def _fetch_ashby(slug: str, company: str, keywords: list[str]) -> list[dict]:
try:
url = f"https://api.ashbyhq.com/posting-api/job-board/{slug}"
- async with httpx.AsyncClient(timeout=8.0) as client:
+ async with http_client.async_client(timeout=8.0) as client:
resp = await client.get(url)
resp.raise_for_status()
data = resp.json()
@@ -620,7 +780,7 @@ async def _fetch_ashby(slug: str, company: str, keywords: list[str]) -> list[dic
async def _fetch_lever(slug: str, company: str, keywords: list[str]) -> list[dict]:
try:
url = f"https://api.lever.co/v0/postings/{slug}?mode=json"
- async with httpx.AsyncClient(timeout=8.0) as client:
+ async with http_client.async_client(timeout=8.0) as client:
resp = await client.get(url)
resp.raise_for_status()
data = resp.json()
@@ -649,7 +809,7 @@ async def _fetch_lever(slug: str, company: str, keywords: list[str]) -> list[dic
async def _fetch_workable(slug: str, company: str, keywords: list[str]) -> list[dict]:
try:
url = f"https://apply.workable.com/api/v1/widget/accounts/{slug}?details=true"
- async with httpx.AsyncClient(timeout=8.0) as client:
+ async with http_client.async_client(timeout=8.0) as client:
resp = await client.get(url)
resp.raise_for_status()
data = resp.json()
@@ -689,7 +849,7 @@ async def _fetch_smartrecruiters(slug: str, company: str, keywords: list[str]) -
jobs: list[dict] = []
limit = 100
offset = 0
- async with httpx.AsyncClient(timeout=10.0) as client:
+ async with http_client.async_client(timeout=10.0) as client:
while offset < 200: # cap at 200 to avoid hammering
url = (
f"https://api.smartrecruiters.com/v1/companies/{slug}/postings"
@@ -949,7 +1109,7 @@ async def _search_level3_websearch() -> list[dict]:
logger.debug("Level 3: BRAVE_SEARCH_API_KEY not set, skipping")
return []
- async with httpx.AsyncClient() as client:
+ async with http_client.async_client() as client:
results = await asyncio.gather(
*[_brave_search_one(client, q) for q in _SEARCH_QUERIES],
return_exceptions=True,
@@ -977,7 +1137,7 @@ async def _search_level4_workable(query: str, location: str) -> list[dict]:
q = urllib.parse.quote(query)
loc = urllib.parse.quote(location)
url = f"https://jobs.workable.com/api/v1/jobs?query={q}&location={loc}"
- async with httpx.AsyncClient(timeout=10.0) as client:
+ async with http_client.async_client(timeout=10.0) as client:
resp = await client.get(url)
resp.raise_for_status()
data = resp.json()
@@ -1017,14 +1177,17 @@ async def _search_level4_adzuna(query: str, location: str) -> list[dict]:
return []
# Map location to country code
+ # Whole-word matching: a substring test sent "Australia" (contains "us") to the US index.
country = "gb"
loc_lower = location.lower()
- if any(x in loc_lower for x in ["usa", "us", "san francisco", "new york", "seattle"]):
- country = "us"
- elif any(x in loc_lower for x in ["canada", "toronto", "vancouver"]):
- country = "ca"
- elif any(x in loc_lower for x in ["australia", "sydney"]):
+ def _mentions(*names: str) -> bool:
+ return any(re.search(rf"(?<![a-z]){re.escape(n)}(?![a-z])", loc_lower) for n in names)
+ if _mentions("australia", "sydney", "melbourne"):
country = "au"
+ elif _mentions("canada", "toronto", "vancouver"):
+ country = "ca"
+ elif _mentions("usa", "us", "united states", "san francisco", "new york", "seattle"):
+ country = "us"
base_url = f"https://api.adzuna.com/v1/api/jobs/{country}/search/1"
query = re.sub(r"\s+", " ", _BOOLEAN_OPS.sub(" ", query)).strip()
@@ -1038,7 +1201,7 @@ async def _fetch(q: str) -> list[dict]:
"results_per_page": 20,
"sort_by": "relevance",
}
- async with httpx.AsyncClient(timeout=12.0) as client:
+ async with http_client.async_client(timeout=12.0) as client:
resp = await client.get(base_url, params=params)
resp.raise_for_status()
data = resp.json()
@@ -1102,7 +1265,7 @@ async def search_jobs_multi(query: str, location: str) -> list[dict]:
l1, l2, l3, l4 = await asyncio.gather(
_search_level1_playwright(keywords),
- _search_level2_ats_apis(query),
+ _search_level2_ats_apis([query]),
_search_level3_websearch(),
_search_level4_adzuna(query, location),
)
@@ -1112,7 +1275,7 @@ async def search_jobs_multi(query: str, location: str) -> list[dict]:
if not deduped:
logger.warning("All live sources returned 0 results — falling back to fixture")
- return _FIXTURE
+ return [dict(j) for j in _FIXTURE] # copies: callers mutate jobs
# Apply title filter — fall back to unfiltered if filter removes everything
filtered = [j for j in deduped if _passes_title_filter(j["title"])]
diff --git a/api/app/jobs_db.py b/api/app/jobs_db.py
index 94bd036..bf1b4ac 100644
--- a/api/app/jobs_db.py
+++ b/api/app/jobs_db.py
@@ -119,26 +119,83 @@ def _extract_tags(job: dict) -> str:
from . import semantic
+# Tags recognised in the `tags` filter (must match what _extract_tags writes). Role tags are
+# OR'd ("ml or data"); domain tags are AND'd ("fintech and sponsorship"). Any other tag is
+# free text — the search box sends typed terms as tags — and must appear in the job's
+# title, company or description.
+ROLE_TAGS = frozenset({"software", "ml", "ai", "data", "backend", "frontend", "fullstack",
+ "devops", "security", "analyst", "engineer", "developer"})
+DOMAIN_TAGS = frozenset({"fintech", "startup", "sponsorship", "remote"})
+
+# Most-recent matching rows that are ranked per request. Bounds per-request work (rows,
+# vector BLOBs, numpy) as the table grows; matches older than this aren't reachable.
+_CANDIDATE_POOL = 2000
+
+# Country-level location aliases. SQL uses them as a LIKE superset; the precise check is a
+# word-boundary regex in Python (so 'us' matches "Austin, TX, US" but not "Australia").
+_LOCATION_ALIASES = {
+ "uk": ["uk", "united kingdom", "england", "scotland", "wales", "northern ireland", "london", "bristol",
+ "manchester", "birmingham", "leeds", "edinburgh", "glasgow", "cardiff", "belfast"],
+ "us": ["us", "usa", "united states", "new york", "san francisco", "california", "seattle"],
+}
+_LOCATION_ALIAS_KEYS = {"uk": "uk", "united kingdom": "uk", "gb": "uk", "great britain": "uk",
+ "us": "us", "usa": "us", "united states": "us", "america": "us"}
+_REMOTE_TERMS = ["remote", "anywhere", "telecommute"]
+
+
+def _job_key(job: dict) -> tuple[str, str, str]:
+ """The jobs table's dedup key (idx_dedup_jobs)."""
+ return (_normalize_company(job.get("company", "")), (job.get("title") or "").lower(),
+ (job.get("location") or "").lower())
+
+
+def _job_text(job: dict) -> str:
+ return f"{job.get('title', '')} {job.get('description', '')}"
+
+
def add_jobs(jobs: list[dict]) -> tuple[int, int]:
- """Insert a list of jobs into the DB, ignoring duplicates."""
- added = 0
- updated = 0
+ """
+ Upsert jobs: insert new ones, refresh updated_at/tags on existing ones. Embeddings are only
+ computed for rows that need one (new, or stored without a vector), in batched Ollama calls,
+ and with no DB connection held — so a slow embedder never holds the write lock.
+ Blocking: call via asyncio.to_thread() from async code.
+ """
if not _DB_PATH:
init_db()
-
+
+ # 1. Classify against what's stored (read-only).
con = sqlite3.connect(_DB_PATH)
+ new_jobs: list[dict] = []
+ existing: list[tuple[str, dict, bool]] = [] # (row id, job, needs_vector)
+ seen: set[tuple[str, str, str]] = set()
for job in jobs:
- now_str = _now()
- normalized_co = _normalize_company(job.get("company", ""))
- tags_json = _extract_tags(job)
-
- # Calculate semantic vector for the job (title + desc)
- job_text = f"{job.get('title', '')} {job.get('description', '')}"
- embedding = semantic.get_embedding_sync(job_text)
- vector_blob = None
- if embedding:
- vector_blob = np.array(embedding, dtype=np.float32).tobytes()
-
+ key = _job_key(job)
+ if key in seen: # duplicate within this batch
+ continue
+ seen.add(key)
+ row = con.execute(
+ "SELECT id, semantic_vector IS NULL FROM jobs "
+ "WHERE lower(normalized_company)=? AND lower(title)=? AND lower(location)=?", key,
+ ).fetchone()
+ if row:
+ existing.append((row[0], job, bool(row[1])))
+ else:
+ new_jobs.append(job)
+ con.close()
+
+ # 2. Embed only what needs it (no DB handle open).
+ to_embed = new_jobs + [job for _, job, needs in existing if needs]
+ vectors = semantic.get_embeddings_sync([_job_text(j) for j in to_embed]) if to_embed else []
+ blob_for = {
+ id(j): (np.array(v, dtype=np.float32).tobytes() if v else None)
+ for j, v in zip(to_embed, vectors)
+ }
+
+ # 3. Write.
+ added = updated = 0
+ now_str = _now()
+ con = sqlite3.connect(_DB_PATH)
+ for job in new_jobs:
try:
con.execute(
"""INSERT INTO jobs
@@ -146,42 +203,30 @@ def add_jobs(jobs: list[dict]) -> tuple[int, int]:
salary_min, salary_max, exposure_score, level, source, tags, semantic_vector, created_at, updated_at, expires_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(
- uuid.uuid4().hex,
- str(job.get("job_id", "")),
- job.get("title", ""),
- job.get("company", ""),
- normalized_co,
- job.get("location", ""),
- job.get("description", ""),
- job.get("url", ""),
- job.get("salary_min"),
- job.get("salary_max"),
- job.get("exposure_score"),
- job.get("level", "mid"),
- job.get("source", "unknown"),
- tags_json,
- vector_blob,
- now_str,
- now_str,
- job.get("expires_at"),
+ uuid.uuid4().hex, str(job.get("job_id", "")), job.get("title", ""), job.get("company", ""),
+ _normalize_company(job.get("company", "")), job.get("location", ""), job.get("description", ""),
+ job.get("url", ""), job.get("salary_min"), job.get("salary_max"), job.get("exposure_score"),
+ job.get("level", "mid"), job.get("source", "unknown"), _extract_tags(job), blob_for.get(id(job)),
+ now_str, now_str, job.get("expires_at"),
),
)
added += 1
- except sqlite3.IntegrityError:
- con.execute(
- """UPDATE jobs SET updated_at = ?, tags = ?,
- semantic_vector = COALESCE(semantic_vector, ?)
- WHERE lower(normalized_company)=? AND lower(title)=lower(?) AND lower(location)=lower(?)""",
- (now_str, tags_json, vector_blob, normalized_co, job.get("title", ""), job.get("location", ""))
- )
- updated += 1
+ except sqlite3.IntegrityError: # inserted concurrently since step 1 — refresh instead
+ existing.append(("", job, False))
except Exception as exc:
logger.error("Failed to insert job %s: %s", job.get("title"), exc)
-
+ for row_id, job, _ in existing:
+ con.execute(
+ """UPDATE jobs SET updated_at = ?, tags = ?, semantic_vector = COALESCE(semantic_vector, ?)
+ WHERE lower(normalized_company)=? AND lower(title)=? AND lower(location)=?""",
+ (now_str, _extract_tags(job), blob_for.get(id(job)), *_job_key(job)),
+ )
+ updated += 1
con.commit()
con.close()
return added, updated
+
def purge_expired_jobs(sync_start: str) -> None:
if not _DB_PATH: return
con = sqlite3.connect(_DB_PATH)
@@ -192,6 +237,30 @@ def purge_expired_jobs(sync_start: str) -> None:
if deleted > 0:
logger.info("Purged %d expired jobs that were removed from their source ATS.", deleted)
+
+def _like(term: str) -> str:
+ """LIKE pattern for a user-supplied term with %, _ and the escape char escaped (ESCAPE '\\')."""
+ return "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+
+
+def _location_matcher(location: str):
+ """(sql_terms, precise_regex) for a location filter."""
+ loc = location.lower().strip()
+ terms = _LOCATION_ALIASES.get(_LOCATION_ALIAS_KEYS.get(loc, ""), [loc])
+ regex = re.compile(r"(?<![a-z])(" + "|".join(re.escape(t) for t in terms) + r")(?![a-z])")
+ return terms, regex
+
+
+def _parse_tags(tags: list[str]) -> tuple[list[str], list[str], list[str]]:
+ roles, domains, text = [], [], []
+ for t in tags or []:
+ t = t.strip().lower()
+ if not t:
+ continue
+ (roles if t in ROLE_TAGS else domains if t in DOMAIN_TAGS else text).append(t)
+ return roles, domains, text
+
+
def get_jobs(
query: str = "",
location: str = "",
@@ -201,91 +270,135 @@ def get_jobs(
tags: list[str] = None,
persona_vector: list[float] = None,
limit: int = 20,
- offset: int = 0
-) -> list[dict]:
+ offset: int = 0,
+ exclude: set[tuple[str, str]] | None = None,
+) -> tuple[list[dict], bool]:
+ """
+ Filter in SQL, then rank (semantic similarity to `persona_vector` when given, else
+ recency) and paginate. `query`, `category` and free-text `tags` are hard filters; role
+ tags are OR'd, domain tags AND'd. `exclude` holds (lower company, lower title) pairs to
+ drop *before* pagination (jobs the user already applied to / discarded), so pages stay
+ full. Returns (jobs, has_more). Blocking: call via asyncio.to_thread() from async code.
+ """
if not _DB_PATH:
init_db()
-
- con = sqlite3.connect(_DB_PATH)
- con.row_factory = sqlite3.Row
-
- # Categorise tags for Collection (OR) logic within Role group
- # Level is now handled by the separate dropdown, not by tags.
- s_roles = {"software", "ml", "ai", "data", "backend", "frontend", "fullstack", "devops", "security", "analyst", "engineer", "developer"}
-
- active_roles = []
- active_others = []
-
- if tags:
- for t in tags:
- t_low = t.lower()
- if t_low in s_roles: active_roles.append(t_low)
- else: active_others.append(t_low)
-
- sql = "SELECT * FROM jobs WHERE 1=1"
- params = []
- sql += " AND (expires_at IS NULL OR expires_at > ?)"
- params.append(_now())
-
- # LEVEL and LOCATION remain strict "Hard Filters" (Deal-breakers)
+
+ roles, domains, text_terms = _parse_tags(list(tags or []) + ([category] if category else []))
+ text_terms += [w for w in re.split(r"[\s,]+", (query or "").lower()) if len(w) > 1]
+
+ where = ["(expires_at IS NULL OR expires_at > ?)"]
+ params: list = [_now()]
+
if level:
- sql += " AND lower(level) = ?"
+ where.append("lower(level) = ?")
params.append(level.lower())
-
- if location:
- loc_lower = location.lower().strip()
- if "london" in loc_lower and "uk" in loc_lower:
- sql += " AND lower(location) NOT LIKE '%ontario%' AND lower(location) NOT LIKE '%new london%'"
-
- if loc_lower == "remote" or remote:
- sql += " AND (lower(location) LIKE '%remote%' OR lower(location) LIKE '%anywhere%' OR lower(location) LIKE '%telecommute%')"
+
+ for term in text_terms:
+ where.append("(lower(title) LIKE ? ESCAPE '\\' OR lower(company) LIKE ? ESCAPE '\\' "
+ "OR lower(description) LIKE ? ESCAPE '\\')")
+ params += [_like(term)] * 3
+ if roles:
+ where.append("(" + " OR ".join("tags LIKE ?" for _ in roles) + ")")
+ params += [f'%"{r}"%' for r in roles]
+ for d in domains:
+ where.append("tags LIKE ?")
+ params.append(f'%"{d}"%')
+
+ loc_regex = None
+ loc_lower = location.lower().strip()
+ remote_only = remote or loc_lower == "remote"
+ remote_sql = "(" + " OR ".join("lower(location) LIKE ?" for _ in _REMOTE_TERMS) + ")"
+ if remote_only and not (location and loc_lower != "remote"):
+ where.append(remote_sql)
+ params += [f"%{t}%" for t in _REMOTE_TERMS]
+ elif location:
+ terms, loc_regex = _location_matcher(location)
+ loc_sql = "(" + " OR ".join("lower(location) LIKE ? ESCAPE '\\'" for _ in terms) + ")"
+ if remote: # "in <location>, or remote"
+ where.append(f"({loc_sql} OR {remote_sql})")
+ params += [_like(t) for t in terms] + [f"%{t}%" for t in _REMOTE_TERMS]
else:
- loc_clause = ""
- if loc_lower in ["uk", "united kingdom", "gb", "great britain"]:
- loc_clause = "(lower(location) LIKE '%uk%' OR lower(location) LIKE '%united kingdom%' OR lower(location) LIKE '%england%' OR lower(location) LIKE '%london%' OR lower(location) LIKE '%bristol%' OR lower(location) LIKE '%manchester%' OR lower(location) LIKE '%birmingham%' OR lower(location) LIKE '%leeds%' OR lower(location) LIKE '%scotland%' OR lower(location) LIKE '%edinburgh%' OR lower(location) LIKE '%glasgow%' OR lower(location) LIKE '%wales%' OR lower(location) LIKE '%cardiff%' OR lower(location) LIKE '%northern ireland%' OR lower(location) LIKE '%belfast%')"
- elif loc_lower in ["us", "usa", "united states", "america"]:
- loc_clause = "(lower(location) LIKE '%us%' OR lower(location) LIKE '%usa%' OR lower(location) LIKE '%united states%' OR lower(location) LIKE '%new york%' OR lower(location) LIKE '%san francisco%' OR lower(location) LIKE '%california%' OR lower(location) LIKE '%seattle%')"
- else:
- loc_clause = "(lower(location) LIKE ?)"
- params.append(f"%{loc_lower}%")
-
- if remote:
- sql += f" AND ({loc_clause} OR lower(location) LIKE '%remote%')"
- else:
- sql += f" AND {loc_clause}"
-
- # Roles and Domains are now "Semantic Interests" handled in Python ranking,
- # so we no longer apply them in SQL.
-
- # To keep semantic search snappy but accurate, we pull more than the limit
- # then rank and return the page. Capping at 500 for performance.
- fetch_limit = 500 if persona_vector else limit + offset
- sql += f" LIMIT {fetch_limit}"
-
- rows = con.execute(sql, params).fetchall()
+ where.append(loc_sql)
+ params += [_like(t) for t in terms]
+ # "London" (or the UK alias, which includes it) means London, England — unless the
+ # user is explicitly searching for the Canadian / US ones.
+ if ("london" in terms) and not any(x in loc_lower for x in ("ontario", "new london", "canada")):
+ where.append("lower(location) NOT LIKE '%ontario%' AND lower(location) NOT LIKE '%new london%'")
+
+ columns = "*" if persona_vector else (
+ "id, job_id, title, company, normalized_company, location, description, url, salary_min, salary_max, "
+ "exposure_score, level, source, tags, created_at, updated_at, expires_at")
+ sql = (f"SELECT {columns} FROM jobs WHERE {' AND '.join(where)} "
+ f"ORDER BY coalesce(updated_at, created_at) DESC LIMIT {_CANDIDATE_POOL}")
+
+ con = sqlite3.connect(_DB_PATH)
+ con.row_factory = sqlite3.Row
+ rows = [dict(r) for r in con.execute(sql, params).fetchall()]
con.close()
-
+
+ exclude = exclude or set()
candidates = []
- for row in rows:
- job = dict(row)
- sim = 0.0
- if persona_vector and job.get("semantic_vector"):
- try:
- job_v = np.frombuffer(job["semantic_vector"], dtype=np.float32).tolist()
- sim = semantic.cosine_similarity(persona_vector, job_v)
- except: pass
-
- job.pop("semantic_vector", None)
- job["semantic_score"] = round(sim, 3)
+ for job in rows:
+ if exclude and ((job.get("company") or "").lower(), (job.get("title") or "").lower()) in exclude:
+ continue
+ loc = (job.get("location") or "").lower()
+ if loc_regex is not None and not loc_regex.search(loc):
+ if not (remote and any(t in loc for t in _REMOTE_TERMS)):
+ continue
candidates.append(job)
- # Sort and Limit
if persona_vector:
- candidates.sort(key=lambda x: x.get("semantic_score", 0), reverse=True)
+ _rank_semantically(candidates, persona_vector)
+ candidates.sort(key=lambda j: j["semantic_score"], reverse=True) # stable: ties stay newest-first
else:
- candidates.sort(key=lambda x: x.get("created_at", ""), reverse=True)
-
- return candidates[offset : offset + limit]
+ for job in candidates:
+ job["semantic_score"] = 0.0
+
+ page = candidates[offset: offset + limit]
+ return page, len(candidates) > offset + limit
+
+
+def _rank_semantically(jobs: list[dict], persona_vector: list[float]) -> None:
+ """Set job['semantic_score'] = cosine(job vector, persona vector), vectorised with numpy."""
+ p = np.asarray(persona_vector, dtype=np.float32)
+ p_norm = float(np.linalg.norm(p)) or 1.0
+ idx, vecs = [], []
+ for i, job in enumerate(jobs):
+ blob = job.pop("semantic_vector", None)
+ job["semantic_score"] = 0.0
+ if blob:
+ v = np.frombuffer(blob, dtype=np.float32)
+ if v.shape == p.shape: # skip vectors from a different embedding model
+ idx.append(i)
+ vecs.append(v)
+ if not vecs:
+ return
+ m = np.vstack(vecs)
+ norms = np.linalg.norm(m, axis=1)
+ norms[norms == 0] = 1.0
+ sims = (m @ p) / (norms * p_norm)
+ for i, s in zip(idx, sims):
+ jobs[i]["semantic_score"] = round(float(s), 3)
+
+
+def get_job(job_db_id: str) -> dict | None:
+ """Return the stored job row (without its embedding) by primary key, or None."""
+ if not _DB_PATH:
+ init_db()
+ try:
+ con = sqlite3.connect(_DB_PATH)
+ con.row_factory = sqlite3.Row
+ row = con.execute("SELECT * FROM jobs WHERE id = ?", (job_db_id,)).fetchone()
+ con.close()
+ except Exception as exc:
+ logger.error("Failed to load job %s: %s", job_db_id, exc)
+ return None
+ if not row:
+ return None
+ job = dict(row)
+ job.pop("semantic_vector", None)
+ return job
+
def update_description(job_db_id: str, description: str) -> None:
"""Persist a freshly-fetched full description back onto a stored job row."""
diff --git a/api/app/main.py b/api/app/main.py
index 2335190..f5d711b 100644
--- a/api/app/main.py
+++ b/api/app/main.py
@@ -1,10 +1,11 @@
import asyncio
import json
import logging
+import os
import pathlib
-import tempfile
+import re
import uuid
-from contextlib import asynccontextmanager
+from contextlib import aclosing, asynccontextmanager
from typing import Optional, Annotated
from fastapi import FastAPI, UploadFile, File, Form, HTTPException, Query, Depends, status
@@ -13,14 +14,14 @@
from fastapi.security import OAuth2PasswordRequestForm
from .cv_parser import extract_text
-from .agent import run_pipeline, run_apply, execute_tool
-from .guardrails import check_cv_for_injection, get_audit_log, GuardrailViolation
-from .job_proxy import search_jobs
+from .agent import run_apply, execute_tool
+from .guardrails import check_cv_for_injection, GuardrailViolation
from . import tracker
from . import auth
from . import pdf_generator
from . import doc_render
from . import jobs_db
+from . import net_guard
from .models import Persona
@@ -39,6 +40,8 @@ async def lifespan(app: FastAPI):
jobs_db.init_db()
pdf_generator.init_pdf_dir(data_dir)
yield
+ from . import discovery
+ discovery.shutdown()
app = FastAPI(title="Windrush API", lifespan=lifespan)
@@ -50,11 +53,14 @@ async def lifespan(app: FastAPI):
allow_headers=["*"],
)
-# Session registries (now also keyed by user_id for better isolation)
+# Live browser-apply sessions
_browser_queues: dict[str, asyncio.Queue] = {} # session_id → instruction queue
_browser_frames: dict[str, asyncio.Queue] = {} # session_id → CDP screencast frame queue
-_cv_files: dict[str, str] = {} # cv_session_id → temp file path
-_cv_texts: dict[str, str] = {} # cv_session_id → extracted text
+
+
+def _original_cv_path(user_id: str) -> pathlib.Path:
+ """The user's most recently uploaded CV — one file per user, overwritten on re-upload."""
+ return pathlib.Path(os.environ.get("APP_DATA_PATH", "/tmp")) / "cvs" / f"{user_id}.pdf"
# ── Auth Endpoints ────────────────────────────────────────────────────────────
@@ -88,10 +94,17 @@ async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]):
# ── Persona Endpoints ─────────────────────────────────────────────────────────
+# The job-site password is stored encrypted outside the persona and never sent back to the
+# browser. GET returns this placeholder when one is saved; PUT treats it as "keep as-is",
+# '' as "clear", and anything else as a new password.
+JOB_PASSWORD_MASK = "••••••••"
+
+
@app.get("/persona", response_model=Persona)
async def get_persona(current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
- data = tracker.get_user_persona(current_user.id)
- return Persona(**data)
+ persona = Persona(**tracker.get_user_persona(current_user.id))
+ persona.core_info.job_password = JOB_PASSWORD_MASK if tracker.has_job_password(current_user.id) else ""
+ return persona
@app.put("/persona")
@@ -99,12 +112,55 @@ async def update_persona(
persona: Persona,
current_user: Annotated[auth.User, Depends(auth.get_current_user)],
):
+ password = persona.core_info.job_password
+ if password != JOB_PASSWORD_MASK:
+ tracker.set_job_password(current_user.id, password)
+ persona.core_info.job_password = ""
ok = tracker.update_user_persona(current_user.id, persona.model_dump())
if not ok:
raise HTTPException(status_code=500, detail="Failed to update persona")
return {"message": "Persona updated"}
+@app.get("/persona/export")
+async def export_persona(
+ current_user: Annotated[auth.User, Depends(auth.get_current_user)],
+ format: str = Query("json", pattern="^(json|md|pdf)$"),
+):
+ """Download the user's persona twin as JSON, Markdown, or a formatted CV PDF."""
+ from fastapi.responses import JSONResponse, Response
+ from .persona_export import persona_to_markdown
+ from .agent import _persona_to_cvdoc
+
+ persona = tracker.get_user_persona(current_user.id)
+
+ if format == "json":
+ return JSONResponse(
+ content=persona,
+ headers={"Content-Disposition": 'attachment; filename="windrush_persona.json"'},
+ )
+
+ if format == "md":
+ return Response(
+ content=persona_to_markdown(persona),
+ media_type="text/markdown; charset=utf-8",
+ headers={"Content-Disposition": 'attachment; filename="windrush_persona.md"'},
+ )
+
+ # format == "pdf" — render the persona through the existing CV pipeline.
+ cvdoc = _persona_to_cvdoc(persona)
+ try:
+ doc_id = await doc_render.render_pdf("cv", cvdoc, "classic")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"PDF render failed: {exc}")
+ path = pdf_generator.get_pdf_path(doc_id)
+ return FileResponse(
+ path,
+ media_type="application/pdf",
+ headers={"Content-Disposition": 'attachment; filename="windrush_persona.pdf"'},
+ )
+
+
# ── Protected Endpoints ───────────────────────────────────────────────────────
@app.get("/health")
@@ -112,17 +168,12 @@ async def health():
return {"status": "ok"}
-@app.get("/guardrails/audit")
-async def guardrails_audit(current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
- return get_audit_log()
-
-
@app.post("/upload")
async def upload_cv(
file: UploadFile = File(...),
current_user: Annotated[auth.User, Depends(auth.get_current_user)] = None,
):
- """Parse CV, merge into Persona, and return a cv_session_id for the pipeline."""
+ """Parse a CV PDF, merge what it contains into the Persona, and keep the PDF for applications."""
pdf_bytes = await file.read()
try:
cv_text = extract_text(pdf_bytes)
@@ -135,7 +186,6 @@ async def upload_cv(
raise HTTPException(status_code=422, detail=f"GUARDRAIL: {e.detail}")
# Extract structured data from CV
- from .agent import execute_tool
cv_data = await execute_tool("extract_cv_profile", {"cv_text": cv_text})
# Merge into existing Persona
@@ -225,48 +275,12 @@ async def upload_cv(
tracker.update_user_persona(current_user.id, persona.model_dump())
- cv_session_id = uuid.uuid4().hex
- cv_path = pathlib.Path(tempfile.gettempdir()) / f"cv_{cv_session_id}.pdf"
- cv_path.write_bytes(pdf_bytes)
- _cv_files[cv_session_id] = str(cv_path)
- _cv_texts[cv_session_id] = cv_text
-
- return {"cv_session_id": cv_session_id, "persona": persona}
-
-
-@app.post("/stream")
-async def pipeline_stream(
- cv_session_id: str = Form(...),
- location: str = Form(default="London"),
- current_user: Annotated[auth.User, Depends(auth.get_current_user)] = None,
-):
- cv_text = _cv_texts.get(cv_session_id)
- if not cv_text:
- raise HTTPException(status_code=404, detail="CV session not found or expired — please re-upload")
+ # Keep the original PDF so the apply flow can upload it when no tailored CV is chosen.
+ cv_file = _original_cv_path(current_user.id)
+ cv_file.parent.mkdir(parents=True, exist_ok=True)
+ cv_file.write_bytes(pdf_bytes)
- cv_path_str = _cv_files.get(cv_session_id, "")
-
- async def pipeline_with_cv_session():
- import time
- try:
- yield f"data: {json.dumps({'type': 'cv_session', 'cv_session_id': cv_session_id, 'timestamp': time.time()})}\n\n"
- async for chunk in run_pipeline(current_user.id, cv_text, location):
- yield chunk
- finally:
- if cv_path_str:
- pathlib.Path(cv_path_str).unlink(missing_ok=True)
- _cv_files.pop(cv_session_id, None)
- _cv_texts.pop(cv_session_id, None)
-
- return StreamingResponse(
- pipeline_with_cv_session(),
- media_type="text/event-stream",
- headers={
- "Cache-Control": "no-cache",
- "X-Accel-Buffering": "no",
- "Connection": "keep-alive",
- },
- )
+ return {"persona": persona}
@app.post("/apply")
@@ -278,7 +292,6 @@ async def apply(
skill_risks: str = Form(default="[]"),
job_email: str = Form(default=""),
job_password: str = Form(default=""),
- cv_session_id: str = Form(default=""),
tailored_cv: str = Form(default=""),
cv_doc_id: str = Form(default=""),
job_title: str = Form(default=""),
@@ -304,19 +317,26 @@ async def apply(
except json.JSONDecodeError:
gaps = []
+ # The browser agent navigates here from inside our network — refuse internal targets.
+ if job_url:
+ try:
+ await net_guard.assert_public_url(job_url)
+ except net_guard.UnsafeURLError as exc:
+ raise HTTPException(status_code=400, detail=f"Job URL not allowed: {exc}")
+
# Fall back to persona credentials if not provided in form
persona = tracker.get_user_persona(current_user.id)
core = persona.get("core_info", {})
effective_email = job_email or core.get("job_email", "")
- effective_password = job_password or core.get("job_password", "")
+ effective_password = job_password or tracker.get_job_password(current_user.id)
- # Tailored CV PDF takes precedence over original upload if the user chose it
- if cv_doc_id:
- tailored_path = pdf_generator.get_pdf_path(cv_doc_id)
- import os as _os2
- cv_path = tailored_path if _os2.path.exists(tailored_path) else _cv_files.get(cv_session_id, "")
+ # Tailored CV PDF (if the user chose it) takes precedence over their uploaded original.
+ original_cv = _original_cv_path(current_user.id)
+ tailored_path = pdf_generator.get_pdf_path(cv_doc_id) if re.fullmatch(r"[a-f0-9]{32}", cv_doc_id) else ""
+ if tailored_path and os.path.exists(tailored_path):
+ cv_path = tailored_path
else:
- cv_path = _cv_files.get(cv_session_id, "")
+ cv_path = str(original_cv) if original_cv.exists() else ""
session_id = str(uuid.uuid4())
job_dict = {
@@ -335,7 +355,7 @@ async def apply(
}
app_id = tracker.add_application(
current_user.id, job_dict, profile, cover_letter, score_data, tailored_cv=tailored_cv
- )
+ ) or tracker.find_application_id(current_user.id, company, job_title) # already saved/tracked
q: asyncio.Queue = asyncio.Queue()
fq: asyncio.Queue = asyncio.Queue(maxsize=8)
@@ -345,22 +365,24 @@ async def apply(
async def cleanup_gen():
user_confirmed = False
try:
- async for chunk in run_apply(
+ # aclosing: if the client disconnects, close run_apply (and the browser beneath it) now
+ async with aclosing(run_apply(
current_user.id, job_id, job_url, cover_letter, risks, session_id, q, fq,
job_email=effective_email, job_password=effective_password, cv_path=cv_path,
- ):
- try:
- payload = json.loads(chunk.removeprefix("data: ").strip())
- if payload.get("type") == "done":
- user_confirmed = True
- except Exception:
- pass
- yield chunk
+ )) as chunks:
+ async for chunk in chunks:
+ try:
+ payload = json.loads(chunk.removeprefix("data: ").strip())
+ if payload.get("type") == "done" and payload.get("submitted") is True:
+ user_confirmed = True
+ except Exception:
+ pass
+ yield chunk
finally:
_browser_queues.pop(session_id, None)
_browser_frames.pop(session_id, None)
if app_id and user_confirmed:
- tracker.update_status(app_id, "Applied")
+ tracker.update_status(app_id, "Applied", user_id=current_user.id)
return StreamingResponse(
cleanup_gen(),
@@ -448,9 +470,11 @@ async def update_application_status(
):
status = body.get("status", "")
notes = body.get("notes")
- ok = tracker.update_status(app_id, status, notes)
- if not ok:
+ if status not in tracker.VALID_STATUSES:
raise HTTPException(status_code=400, detail=f"Invalid status: {status!r}")
+ if not tracker.update_status(app_id, status, notes, user_id=current_user.id):
+ # Unknown id, or another user's application — indistinguishable to the caller by design.
+ raise HTTPException(status_code=404, detail="Application not found")
return {"id": app_id, "status": status, "updated": True}
@@ -501,6 +525,16 @@ async def onboarding_complete(current_user: Annotated[auth.User, Depends(auth.ge
# ── Job Feed ──────────────────────────────────────────────────────────────────
from . import semantic
+from . import discovery
+
+# Live discovery kicks in (in the background) when page 1 has fewer matches than this.
+_THIN_FEED = 5
+# Request-path embedding budget: past this the feed falls back to recency ranking rather than
+# making the user wait on Ollama loading the model (background work still allows 600s).
+_FEED_EMBED_TIMEOUT_S = 20.0
+# Tracker statuses that hide a job from the feed.
+_HIDE_STATUSES = {"Applied", "Responded", "Interview", "Offer", "Rejected", "Discarded"}
+
@app.get("/jobs")
async def get_jobs(
@@ -514,118 +548,57 @@ async def get_jobs(
limit: int = Query(default=20, ge=1, le=100),
current_user: Annotated[auth.User, Depends(auth.get_current_user)] = None,
):
- """Return job listings for the job feed with pagination and filtering."""
- tag_list = [t.strip() for t in tags.split(",")] if tags else []
+ """
+ Job feed. Explicit filters (query, tags, category, level, location, remote) are applied in
+ SQL; the persona only *ranks* (semantic similarity) and supplies a default location. If page
+ 1 comes back thin, live discovery starts in the background and `discovering` tells the
+ client to re-poll — the request itself never waits on scraping or embedding.
+ """
+ tag_list = [t.strip() for t in tags.split(",") if t.strip()] if tags else []
- # Always pull the persona to get preferences and for semantic search
persona_data = tracker.get_user_persona(current_user.id)
-
- effective_query = query
- effective_location = location
-
- if not effective_query:
- prefs = persona_data.get("preferences", {})
- titles = prefs.get("target_titles", [])
- # Join multiple target titles for a broader default search
- effective_query = " ".join(titles) if titles else ""
-
- locs = prefs.get("preferred_locations", [])
- if not effective_location:
- effective_location = locs[0] if locs else ""
-
- # Generate semantic vector that combines Persona + current search query/tags
- # This "Interest-Weighted" vector ensures the AI prioritizes both who you are AND what you want.
+ prefs = persona_data.get("preferences", {}) or {}
+ titles = prefs.get("target_titles", []) or []
+ locs = prefs.get("preferred_locations", []) or []
+ effective_location = location or (locs[0] if locs else "")
+
+ # "Interest-weighted" ranking vector: who you are (persona) + what you're searching for now.
persona_text = semantic.vectorize_persona(persona_data)
-
- # Merge explicit keywords and tags into a single semantic intent block
- intent_parts = []
- if query: intent_parts.append(query)
- if tag_list: intent_parts.extend(tag_list)
-
+ intent_parts = ([query] if query else []) + tag_list
if intent_parts:
intent_str = ", ".join(intent_parts)
- # Boost the query by repeating it in the text
persona_text = f"Current user interest: {intent_str}. Specific focus: {intent_str}. {persona_text}"
-
- persona_vector = await semantic.get_embedding(persona_text)
+ persona_vector = await semantic.get_embedding(persona_text, timeout=_FEED_EMBED_TIMEOUT_S)
- # ── Clean Feed Logic ──
- # Fetch user's active pipeline statuses to filter/badge the feed
status_map = tracker.get_user_application_statuses(current_user.id)
- # Statuses that trigger hiding from the feed
- HIDE_STATUSES = {"Applied", "Responded", "Interview", "Offer", "Rejected", "Discarded"}
-
- def process_clean_feed(raw_jobs: list[dict]) -> list[dict]:
- clean = []
- for job in raw_jobs:
- key = (job.get("company", "").lower(), job.get("title", "").lower())
- status = status_map.get(key)
-
- if status in HIDE_STATUSES:
- continue
-
- if status == "Saved":
- job["tracker_status"] = "Saved"
- elif status:
- # Catch-all for other statuses (e.g. Evaluated, Pending Review)
- job["tracker_status"] = status
-
- clean.append(job)
- return clean
-
- offset = (page - 1) * limit
-
- # Query the local jobs database first
- raw_jobs = jobs_db.get_jobs(
- query=effective_query,
- location=effective_location,
- level=level,
- category=category,
- remote=remote,
- tags=tag_list,
- persona_vector=persona_vector,
- limit=limit,
- offset=offset,
+ hidden = {key for key, status in status_map.items() if status in _HIDE_STATUSES}
+
+ jobs, has_more = await asyncio.to_thread(
+ jobs_db.get_jobs,
+ query=query, location=effective_location, level=level, category=category, remote=remote,
+ tags=tag_list, persona_vector=persona_vector, limit=limit, offset=(page - 1) * limit,
+ exclude=hidden,
)
- jobs = process_clean_feed(raw_jobs)
-
- # Proactive Infinite Discovery: If we don't have enough local matches for this
- # specific location/query, trigger a live discovery across external APIs.
- if (not jobs or len(jobs) < 5):
- # Use only the first title or the specific user query for live search
- live_query = query if query else (titles[0] if persona_data.get("preferences", {}).get("target_titles") else "software engineer")
-
- # Don't trigger if the query is just a single character (e.g. typing)
- if len(live_query) > 2:
- live_jobs = await search_jobs(live_query, effective_location or "London")
- if live_jobs:
- # Persist them for future queries
- for j in live_jobs:
- j["source"] = j.get("source", "live")
- j["level"] = _infer_level(j.get("title", ""))
- jobs_db.add_jobs(live_jobs)
-
- # Re-query and re-filter
- refreshed_raw = jobs_db.get_jobs(
- query=effective_query,
- location=effective_location,
- level=level,
- category=category,
- remote=remote,
- tags=tag_list,
- persona_vector=persona_vector,
- limit=limit,
- offset=offset,
- )
- jobs = process_clean_feed(refreshed_raw)
+ for job in jobs:
+ status = status_map.get(((job.get("company") or "").lower(), (job.get("title") or "").lower()))
+ if status:
+ job["tracker_status"] = status
+
+ discovering = False
+ if page == 1 and len(jobs) < _THIN_FEED:
+ free_text = [t for t in tag_list if t.lower() not in jobs_db.ROLE_TAGS | jobs_db.DOMAIN_TAGS]
+ live_query = query or (free_text[0] if free_text else "") or (titles[0] if titles else "software engineer")
+ if len(live_query) > 2: # don't fire on a stray character
+ discovering = discovery.ensure(live_query, effective_location or "London")
return {
"jobs": jobs,
- "query": effective_query,
+ "query": query,
"location": effective_location,
"page": page,
"limit": limit,
- "has_more": len(jobs) == limit,
+ "has_more": has_more,
+ "discovering": discovering,
}
@@ -654,21 +627,63 @@ async def _ensure_full_description(job: dict) -> dict:
are only a web snippet), so anything that short is re-fetched from the live
listing page. The result is persisted back to the DB so re-runs and revisits
are instant.
+
+ The jobs table is shared by all users, so the client-supplied job is never
+ trusted for the write: we only persist when the id matches a stored row, and
+ then we fetch that row's *stored* URL — never the URL from the request body.
"""
desc = (job.get("description") or "").strip()
if len(desc) > 600: # already a full description (e.g. previously fetched)
return job
from .job_searcher import fetch_full_description
+ db_id = str(job.get("id") or "")
+ stored = jobs_db.get_job(db_id) if db_id else None
+ if stored:
+ stored_desc = (stored.get("description") or "").strip()
+ if len(stored_desc) > 600: # another request already fetched it
+ return {**job, "description": stored_desc}
+ full = await fetch_full_description(stored.get("url") or "")
+ if full and len(full) > len(stored_desc):
+ jobs_db.update_description(db_id, full)
+ return {**job, "description": full}
+ return job
+
+ # Not a stored job (e.g. pasted via /jobs/from-url): enrich this request only.
full = await fetch_full_description(job.get("url", ""))
if full and len(full) > len(desc):
job = {**job, "description": full}
- db_id = job.get("id")
- if db_id:
- jobs_db.update_description(str(db_id), full)
return job
+@app.post("/jobs/from-url")
+async def job_from_url(
+ body: dict,
+ current_user: Annotated[auth.User, Depends(auth.get_current_user)],
+):
+ """
+ Scrape a user-pasted job-posting URL into a full Job object for analysis.
+
+ The job is returned (not persisted) — the frontend keeps it in sessionStorage
+ and runs the normal detail-page flow; it only enters the DB if the user Saves.
+ """
+ from .job_searcher import scrape_job_from_url
+
+ url = (body.get("url") or "").strip()
+ if not url.startswith("http"):
+ raise HTTPException(status_code=400, detail="Please paste a valid http(s) link.")
+ try:
+ job = await scrape_job_from_url(url)
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc))
+ except Exception as exc:
+ raise HTTPException(status_code=502, detail=f"Could not fetch that listing: {exc}")
+
+ job["level"] = _infer_level(job.get("title", ""))
+ job["tags"] = jobs_db._extract_tags(job)
+ return {"job": job}
+
+
@app.post("/jobs/analyze")
async def analyze_job(
body: dict,
diff --git a/api/app/models.py b/api/app/models.py
index 7e2147e..4b2ce1a 100644
--- a/api/app/models.py
+++ b/api/app/models.py
@@ -1,23 +1,5 @@
from pydantic import BaseModel, Field
-from typing import Any, List, Optional
-
-
-class SkillRisk(BaseModel):
- skill: str
- onet_code: str | None = None
- occupation_name: str = ""
- overall_exposure: float = 0.5
-
-
-class CVProfile(BaseModel):
- name: str = ""
- email: str = ""
- phone: str = ""
- location: str = ""
- skills: list[str] = []
- experience_years: float = 0
- job_titles: list[str] = []
- summary: str = ""
+from typing import List, Optional
# ── PERSONA MODELS ────────────────────────────────────────────────────────────
@@ -141,51 +123,3 @@ class Persona(BaseModel):
custom_directives: str = ""
skills: List[SkillCategory] = []
summary: str = ""
-
-
-class JobMatch(BaseModel):
- job_id: str
- title: str
- company: str
- location: str
- description: str = ""
- url: str = ""
- fit_score: float = 0.0
- exposure_score: float = 0.5
- composite_score: float = 0.0
- level_match: str = "ok"
- skill_gaps: list[str] = []
-
-
-class ApplicationRecord(BaseModel):
- id: str
- job_id: str
- job_title: str
- company: str
- location: str = ""
- job_url: str = ""
- status: str = "Evaluated"
- date_applied: str | None = None
- composite_score: float | None = None
- exposure_score: float | None = None
- fit_score: float | None = None
- skill_gaps: list[str] = []
- level_match: str = "ok"
- notes: str = ""
- created_at: str
-
-
-class RoadmapItem(BaseModel):
- skill: str
- action: str
- timeline: str
- resource: str = ""
-
-
-class AgentEvent(BaseModel):
- type: str
- timestamp: float
- tool_name: str | None = None
- tool_input: dict[str, Any] | None = None
- result: Any | None = None
- text: str | None = None
diff --git a/api/app/net_guard.py b/api/app/net_guard.py
new file mode 100644
index 0000000..92bf6a8
--- /dev/null
+++ b/api/app/net_guard.py
@@ -0,0 +1,62 @@
+"""
+SSRF guard for server-side requests to user-influenced URLs.
+
+The API runs inside the Docker network next to ollama, jobs-mcp, civic-guardrails, etc., so
+any URL a user can steer us to (pasted job links, job URLs in request bodies, redirects from
+those pages) must resolve only to public addresses. Every redirect hop is re-checked.
+
+Residual risk: DNS rebinding (the name re-resolving to a private IP between our check and
+httpx's own lookup). Closing that needs IP pinning at the transport layer.
+"""
+import asyncio
+import ipaddress
+import socket
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+_MAX_REDIRECTS = 5
+
+
+class UnsafeURLError(ValueError):
+ """The URL is malformed, non-http(s), or points at a non-public address."""
+
+
+def _is_public(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped:
+ ip = ip.ipv4_mapped
+ return ip.is_global and not ip.is_multicast
+
+
+def check_public_url(url: str) -> None:
+ """Raise UnsafeURLError unless `url` is http(s) and its host resolves only to public IPs."""
+ parsed = urlparse(url or "")
+ if parsed.scheme not in ("http", "https"):
+ raise UnsafeURLError("only http(s) links are allowed")
+ host = parsed.hostname
+ if not host:
+ raise UnsafeURLError("the link has no host")
+ try:
+ port = parsed.port or (443 if parsed.scheme == "https" else 80)
+ infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+ except (socket.gaierror, UnicodeError, ValueError) as exc:
+ raise UnsafeURLError(f"could not resolve {host}") from exc
+ for info in infos:
+ ip = ipaddress.ip_address(info[4][0].split("%", 1)[0]) # drop IPv6 zone id
+ if not _is_public(ip):
+ raise UnsafeURLError(f"{host} resolves to a non-public address")
+
+
+async def assert_public_url(url: str) -> None:
+ await asyncio.to_thread(check_public_url, url) # getaddrinfo blocks
+
+
+async def safe_get(client: httpx.AsyncClient, url: str) -> httpx.Response:
+ """GET `url`, following redirects manually so every hop passes check_public_url."""
+ for _ in range(_MAX_REDIRECTS + 1):
+ await assert_public_url(url)
+ resp = await client.get(url, follow_redirects=False)
+ if not resp.is_redirect:
+ return resp
+ url = urljoin(str(resp.url), resp.headers.get("location", ""))
+ raise UnsafeURLError("too many redirects")
diff --git a/api/app/persona_export.py b/api/app/persona_export.py
new file mode 100644
index 0000000..ab110da
--- /dev/null
+++ b/api/app/persona_export.py
@@ -0,0 +1,191 @@
+"""
+Human-readable Markdown export of a user's Persona ("Digital Twin").
+
+Renders the persona dict (same shape as `models.Persona`) into a clean,
+copy-pasteable Markdown document covering every section the profile editor
+exposes. Missing/empty fields are skipped so the output stays tidy regardless
+of how much of the persona the user has filled in.
+"""
+from __future__ import annotations
+
+
+def _full_name(core: dict) -> str:
+ name = f"{core.get('first_name', '')} {core.get('last_name', '')}".strip()
+ preferred = core.get("preferred_name", "").strip()
+ if preferred and preferred not in name:
+ name = f"{name} ({preferred})".strip()
+ return name
+
+
+def _date_range(start: str, end: str, ongoing: bool) -> str:
+ end_label = "Present" if ongoing else (end or "")
+ parts = [p for p in (start, end_label) if p]
+ return " – ".join(parts)
+
+
+def _join(values, sep: str = ", ") -> str:
+ if isinstance(values, (list, tuple)):
+ return sep.join(str(v) for v in values if str(v).strip())
+ return str(values or "")
+
+
+def persona_to_markdown(persona: dict) -> str:
+ """Render a persona dict as a Markdown document."""
+ core = persona.get("core_info", {}) or {}
+ lines: list[str] = []
+
+ name = _full_name(core) or "Persona Twin"
+ lines.append(f"# {name}")
+ lines.append("")
+
+ summary = (persona.get("summary") or "").strip()
+ if summary:
+ lines.append("## Professional Summary")
+ lines.append("")
+ lines.append(summary)
+ lines.append("")
+
+ # ── Core Info ─────────────────────────────────────────────────────────────
+ contact_rows: list[tuple[str, str]] = [
+ ("Email", core.get("email", "")),
+ ("Phone", core.get("phone", "")),
+ ("Location", ", ".join(p for p in [core.get("city", ""), core.get("country", "")] if p)),
+ ("LinkedIn", core.get("linkedin", "")),
+ ("GitHub", core.get("github", "")),
+ ("Portfolio", core.get("portfolio", "") or core.get("website", "")),
+ ("Visa status", core.get("visa_status", "")),
+ ]
+ contact_rows = [(k, v) for k, v in contact_rows if str(v).strip()]
+ if contact_rows:
+ lines.append("## Core Info")
+ lines.append("")
+ for label, value in contact_rows:
+ lines.append(f"- **{label}:** {value}")
+ lines.append("")
+
+ # ── Preferences ───────────────────────────────────────────────────────────
+ prefs = persona.get("preferences", {}) or {}
+ pref_rows: list[tuple[str, str]] = [
+ ("Target roles", _join(prefs.get("target_titles", []))),
+ ("Min salary", str(prefs.get("min_salary") or "")),
+ ("Remote preference", prefs.get("remote_preference", "")),
+ ("Preferred locations", _join(prefs.get("preferred_locations", []))),
+ ("Industries", _join(prefs.get("industries", []))),
+ ("Employment type", prefs.get("employment_type", "")),
+ ("Notice period", prefs.get("notice_period", "")),
+ ]
+ pref_rows = [(k, v) for k, v in pref_rows if str(v).strip()]
+ if pref_rows:
+ lines.append("## Job Preferences")
+ lines.append("")
+ for label, value in pref_rows:
+ lines.append(f"- **{label}:** {value}")
+ lines.append("")
+
+ # ── Skills ────────────────────────────────────────────────────────────────
+ skills = persona.get("skills", []) or []
+ skill_lines = [
+ f"- **{c.get('category', 'Skills')}:** {_join(c.get('skills', []))}"
+ for c in skills
+ if c.get("skills")
+ ]
+ if skill_lines:
+ lines.append("## Skills")
+ lines.append("")
+ lines.extend(skill_lines)
+ lines.append("")
+
+ # ── Work Experience ───────────────────────────────────────────────────────
+ history = persona.get("history", []) or []
+ if history:
+ lines.append("## Work Experience")
+ lines.append("")
+ for exp in history:
+ title = exp.get("title", "")
+ employer = exp.get("employer", "")
+ header = " — ".join(p for p in [title, employer] if p) or "Role"
+ dates = _date_range(exp.get("start_date", ""), exp.get("end_date", ""), exp.get("is_current", False))
+ lines.append(f"### {header}" + (f" \n*{dates}*" if dates else ""))
+ if exp.get("summary"):
+ lines.append("")
+ lines.append(exp["summary"])
+ achievements = exp.get("achievements", [])
+ if isinstance(achievements, list):
+ for a in achievements:
+ if str(a).strip():
+ lines.append(f"- {a}")
+ elif str(achievements).strip():
+ lines.append(f"- {achievements}")
+ if exp.get("metrics"):
+ lines.append(f"- {_join(exp['metrics'])}")
+ if exp.get("tech_stack"):
+ lines.append(f"- *Tech:* {_join(exp['tech_stack'])}")
+ lines.append("")
+
+ # ── Education ─────────────────────────────────────────────────────────────
+ education = persona.get("education", []) or []
+ if education:
+ lines.append("## Education")
+ lines.append("")
+ for edu in education:
+ degree = edu.get("degree", "")
+ inst = edu.get("institution", "")
+ header = " — ".join(p for p in [degree, inst] if p) or "Education"
+ dates = _date_range(edu.get("start_date", ""), edu.get("end_date", ""), edu.get("is_currently_enrolled", False))
+ detail = " \n".join(p for p in [f"*{dates}*" if dates else "", f"Grade: {edu['grade']}" if edu.get("grade") else ""] if p)
+ lines.append(f"### {header}" + (f" \n{detail}" if detail else ""))
+ lines.append("")
+
+ # ── Projects ──────────────────────────────────────────────────────────────
+ projects = persona.get("projects", []) or []
+ if projects:
+ lines.append("## Projects")
+ lines.append("")
+ for p in projects:
+ name_p = p.get("name", "Project")
+ url = p.get("url")
+ lines.append(f"### {name_p}" + (f" \n{url}" if url else ""))
+ desc = p.get("summary") or p.get("problem_solved") or ""
+ if desc:
+ lines.append("")
+ lines.append(desc)
+ if p.get("outcomes"):
+ lines.append(f"- *Outcomes:* {p['outcomes']}")
+ if p.get("technologies"):
+ lines.append(f"- *Tech:* {_join(p['technologies'])}")
+ lines.append("")
+
+ # ── Certifications ────────────────────────────────────────────────────────
+ certs = persona.get("certifications", []) or []
+ cert_lines = []
+ for c in certs:
+ bits = [c.get("name", ""), c.get("issuing_organization", "")]
+ label = " — ".join(b for b in bits if b)
+ if c.get("issue_date"):
+ label += f" ({c['issue_date']})"
+ if label.strip():
+ cert_lines.append(f"- {label}")
+ if cert_lines:
+ lines.append("## Certifications")
+ lines.append("")
+ lines.extend(cert_lines)
+ lines.append("")
+
+ # ── Story Bank ────────────────────────────────────────────────────────────
+ stories = persona.get("story_bank", []) or []
+ if stories:
+ lines.append("## Story Bank")
+ lines.append("")
+ for s in stories:
+ lines.append(f"### {s.get('title', 'Story')}")
+ if s.get("scenario"):
+ lines.append(f"- **Scenario:** {s['scenario']}")
+ if s.get("action"):
+ lines.append(f"- **Action:** {s['action']}")
+ if s.get("result"):
+ lines.append(f"- **Result:** {s['result']}")
+ if s.get("tags"):
+ lines.append(f"- **Tags:** {_join(s['tags'])}")
+ lines.append("")
+
+ return "\n".join(lines).strip() + "\n"
diff --git a/api/app/semantic.py b/api/app/semantic.py
index e383908..d5ca4da 100644
--- a/api/app/semantic.py
+++ b/api/app/semantic.py
@@ -1,106 +1,150 @@
import logging
import os
-import json
-import numpy as np
+from collections import OrderedDict
from typing import List, Optional
+
import httpx
+from . import http_client
+import numpy as np
+
logger = logging.getLogger("windrush.semantic")
_OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
_EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "nomic-embed-text")
-# Simple in-memory cache for embeddings to avoid redundant calls
-_embedding_cache = {}
+# Background work (sync, job ingestion) tolerates the first call loading the model into
+# limited VRAM; request handlers pass a short timeout and fall back to non-semantic ranking.
+_BACKGROUND_TIMEOUT_S = 600.0
+_BATCH_SIZE = 32
+_MAX_TEXT_CHARS = 6000 # nomic-embed-text's context is ~2k tokens; Ollama truncates beyond it anyway
+
+# Bounded LRU so a long-running API process doesn't grow without limit.
+_CACHE_MAX = 512
+_embedding_cache: "OrderedDict[str, list[float]]" = OrderedDict()
+
+
+def _cache_get(text: str) -> Optional[list[float]]:
+ vec = _embedding_cache.get(text)
+ if vec is not None:
+ _embedding_cache.move_to_end(text)
+ return vec
+
-async def get_embedding(text: str) -> Optional[List[float]]:
- """Get dense vector embedding for text via Ollama."""
+def _cache_put(text: str, vec: list[float]) -> None:
+ _embedding_cache[text] = vec
+ _embedding_cache.move_to_end(text)
+ while len(_embedding_cache) > _CACHE_MAX:
+ _embedding_cache.popitem(last=False)
+
+
+def _clip(text: str) -> str:
+ return (text or "")[:_MAX_TEXT_CHARS]
+
+
+async def get_embedding(text: str, timeout: float = _BACKGROUND_TIMEOUT_S) -> Optional[List[float]]:
+ """Dense vector for one text via Ollama, or None if it's unavailable within `timeout`."""
+ text = _clip(text)
if not text:
return None
-
- if text in _embedding_cache:
- return _embedding_cache[text]
-
+ cached = _cache_get(text)
+ if cached is not None:
+ return cached
try:
- # Increased timeout to 600s to allow for model loading into GPU on limited VRAM
- async with httpx.AsyncClient(timeout=600.0) as client:
- resp = await client.post(
- f"{_OLLAMA_HOST}/api/embeddings",
- json={"model": _EMBEDDING_MODEL, "prompt": text}
- )
- if resp.status_code == 200:
- embedding = resp.json().get("embedding")
- _embedding_cache[text] = embedding
- return embedding
+ async with http_client.async_client(timeout=timeout) as client:
+ resp = await client.post(f"{_OLLAMA_HOST}/api/embed", json={"model": _EMBEDDING_MODEL, "input": [text]})
+ if resp.status_code == 404: # Ollama < 0.3 only has the single-prompt endpoint
+ resp = await client.post(f"{_OLLAMA_HOST}/api/embeddings", json={"model": _EMBEDDING_MODEL, "prompt": text})
+ vec = resp.json().get("embedding") if resp.status_code == 200 else None
else:
- logger.warning(f"Ollama embedding failed with status {resp.status_code}: {resp.text}")
+ vec = (resp.json().get("embeddings") or [None])[0] if resp.status_code == 200 else None
+ if vec:
+ _cache_put(text, vec)
+ return vec
+ logger.warning("Ollama embedding failed with status %s: %s", resp.status_code, resp.text[:200])
except Exception as e:
- logger.error(f"Error getting embedding from Ollama: {e}")
-
+ logger.warning("Error getting embedding from Ollama: %s", e)
return None
+
+def get_embeddings_sync(texts: list[str], timeout: float = _BACKGROUND_TIMEOUT_S) -> list[Optional[list[float]]]:
+ """
+ Embed many texts with batched /api/embed calls (one HTTP round-trip per _BATCH_SIZE texts
+ instead of one per text). Returns a list aligned with `texts`; None where embedding failed.
+ Blocking — call from a worker thread or a background job, never on the event loop.
+ """
+ out: list[Optional[list[float]]] = [None] * len(texts)
+ todo = []
+ for i, t in enumerate(texts):
+ t = _clip(t)
+ cached = _cache_get(t) if t else None
+ if cached is not None:
+ out[i] = cached
+ elif t:
+ todo.append((i, t))
+ if not todo:
+ return out
+ try:
+ with http_client.client(timeout=timeout) as client:
+ for start in range(0, len(todo), _BATCH_SIZE):
+ chunk = todo[start:start + _BATCH_SIZE]
+ resp = client.post(f"{_OLLAMA_HOST}/api/embed",
+ json={"model": _EMBEDDING_MODEL, "input": [t for _, t in chunk]})
+ if resp.status_code == 404: # old Ollama: fall back to one call per text
+ for i, t in chunk:
+ r = client.post(f"{_OLLAMA_HOST}/api/embeddings", json={"model": _EMBEDDING_MODEL, "prompt": t})
+ if r.status_code == 200 and r.json().get("embedding"):
+ out[i] = r.json()["embedding"]
+ continue
+ if resp.status_code != 200:
+ logger.warning("Ollama batch embedding failed with status %s: %s", resp.status_code, resp.text[:200])
+ continue
+ for (i, t), vec in zip(chunk, resp.json().get("embeddings") or []):
+ if vec:
+ out[i] = vec
+ _cache_put(t, vec)
+ except Exception as e:
+ logger.warning("Error getting batch embeddings from Ollama: %s", e)
+ return out
+
+
+def get_embedding_sync(text: str) -> Optional[List[float]]:
+ """Synchronous single-text embedding (see get_embeddings_sync for bulk work)."""
+ return get_embeddings_sync([text])[0]
+
+
def cosine_similarity(v1: List[float], v2: List[float]) -> float:
"""Calculate cosine similarity between two vectors."""
if not v1 or not v2:
return 0.0
-
a = np.array(v1)
b = np.array(v2)
-
- dot = np.dot(a, b)
norm_a = np.linalg.norm(a)
norm_b = np.linalg.norm(b)
-
if norm_a == 0 or norm_b == 0:
return 0.0
-
- return float(dot / (norm_a * norm_b))
+ return float(np.dot(a, b) / (norm_a * norm_b))
-def get_embedding_sync(text: str) -> Optional[List[float]]:
- """Synchronous version of get_embedding for use in db operations."""
- if not text:
- return None
-
- if text in _embedding_cache:
- return _embedding_cache[text]
-
- try:
- # Increased timeout to 600s to allow for model loading into GPU on limited VRAM
- with httpx.Client(timeout=600.0) as client:
- resp = client.post(
- f"{_OLLAMA_HOST}/api/embeddings",
- json={"model": _EMBEDDING_MODEL, "prompt": text}
- )
- if resp.status_code == 200:
- embedding = resp.json().get("embedding")
- _embedding_cache[text] = embedding
- return embedding
- else:
- logger.warning(f"Ollama embedding failed with status {resp.status_code}: {resp.text}")
- except Exception as e:
- logger.error(f"Error getting embedding from Ollama: {e}")
-
- return None
def vectorize_persona(persona: dict) -> str:
"""Create a descriptive string for a persona to be used for embedding."""
parts = []
-
+
# Core titles
prefs = persona.get("preferences", {})
titles = prefs.get("target_titles", [])
if titles:
parts.append(f"Target roles: {', '.join(titles)}")
-
+
# Skills
all_skills = []
for cat in persona.get("skills", []):
all_skills.extend(cat.get("skills", []))
if all_skills:
parts.append(f"Skills: {', '.join(all_skills)}")
-
+
# Experience
for exp in persona.get("history", []):
parts.append(f"{exp.get('title')} at {exp.get('employer')}: {exp.get('summary')}")
-
+
return " ".join(parts)
diff --git a/api/app/tracker.py b/api/app/tracker.py
index 3ae7df8..a517b6c 100644
--- a/api/app/tracker.py
+++ b/api/app/tracker.py
@@ -9,6 +9,8 @@
import uuid
from datetime import datetime, timezone
+from . import crypto
+
logger = logging.getLogger("windrush.tracker")
_DB_PATH: str = ""
@@ -76,11 +78,13 @@ def init_db(db_path: str = "") -> None:
"ALTER TABLE users ADD COLUMN onboarding_complete INTEGER DEFAULT 0",
"ALTER TABLE applications ADD COLUMN tailored_cv TEXT",
"ALTER TABLE applications ADD COLUMN user_id TEXT",
+ "ALTER TABLE users ADD COLUMN job_password_enc TEXT",
]:
try:
con.execute(stmt)
except sqlite3.OperationalError:
pass
+ _migrate_plaintext_job_passwords(con)
# Create index after migrations so user_id column is guaranteed to exist
try:
con.execute(_CREATE_INDEX)
@@ -137,7 +141,34 @@ def get_user_persona(user_id: str) -> dict:
return {}
+def _strip_job_password(persona: dict) -> str:
+ """Remove core_info.job_password from a persona dict in place; return what was there."""
+ core = persona.get("core_info")
+ if isinstance(core, dict):
+ return core.pop("job_password", "") or ""
+ return ""
+
+
+def _migrate_plaintext_job_passwords(con: sqlite3.Connection) -> None:
+ """One-off: move job passwords that older versions stored in the persona JSON into job_password_enc."""
+ rows = con.execute("SELECT id, persona FROM users WHERE persona LIKE '%job_password%'").fetchall()
+ for user_id, raw in rows:
+ try:
+ persona = json.loads(raw or "{}")
+ except json.JSONDecodeError:
+ continue
+ plaintext = _strip_job_password(persona)
+ if plaintext:
+ con.execute("UPDATE users SET job_password_enc=? WHERE id=?", (crypto.encrypt(plaintext), user_id))
+ con.execute("UPDATE users SET persona=? WHERE id=?", (json.dumps(persona), user_id))
+ if rows:
+ logger.info("Moved job passwords for %d user(s) out of persona JSON into encrypted storage", len(rows))
+
+
def update_user_persona(user_id: str, persona: dict) -> bool:
+ # The job-site password never lives in the persona JSON — see set_job_password().
+ persona = json.loads(json.dumps(persona))
+ _strip_job_password(persona)
try:
con = sqlite3.connect(_DB_PATH)
con.execute("UPDATE users SET persona=? WHERE id=?", (json.dumps(persona), user_id))
@@ -149,6 +180,32 @@ def update_user_persona(user_id: str, persona: dict) -> bool:
return False
+def set_job_password(user_id: str, plaintext: str) -> None:
+ """Store (encrypted) or clear ('' ) the user's job-site password."""
+ con = sqlite3.connect(_DB_PATH)
+ con.execute(
+ "UPDATE users SET job_password_enc=? WHERE id=?",
+ (crypto.encrypt(plaintext) if plaintext else None, user_id),
+ )
+ con.commit()
+ con.close()
+
+
+def get_job_password(user_id: str) -> str:
+ """Decrypted job-site password, or '' if none is stored. Only for handing to the browser agent."""
+ con = sqlite3.connect(_DB_PATH)
+ row = con.execute("SELECT job_password_enc FROM users WHERE id=?", (user_id,)).fetchone()
+ con.close()
+ return crypto.decrypt(row[0]) if row and row[0] else ""
+
+
+def has_job_password(user_id: str) -> bool:
+ con = sqlite3.connect(_DB_PATH)
+ row = con.execute("SELECT job_password_enc FROM users WHERE id=?", (user_id,)).fetchone()
+ con.close()
+ return bool(row and row[0])
+
+
def get_onboarding_status(user_id: str) -> bool:
con = sqlite3.connect(_DB_PATH)
row = con.execute("SELECT onboarding_complete FROM users WHERE id=?", (user_id,)).fetchone()
@@ -181,8 +238,10 @@ def add_application(
status = "Pending Review"
app_id = uuid.uuid4().hex
+ # Always close: a failed INSERT leaves its implicit transaction open, and an unclosed
+ # connection would hold the DB write lock ("database is locked" for the next writer).
+ con = sqlite3.connect(_DB_PATH)
try:
- con = sqlite3.connect(_DB_PATH)
con.execute("PRAGMA journal_mode=WAL")
con.execute(
"""INSERT INTO applications
@@ -212,7 +271,6 @@ def add_application(
),
)
con.commit()
- con.close()
logger.info("Tracker: added application %s for user %s (%s @ %s)", app_id, user_id, job.get("title"), job.get("company"))
return app_id
except sqlite3.IntegrityError:
@@ -221,6 +279,8 @@ def add_application(
except Exception as exc:
logger.error("Tracker add_application failed: %s", exc)
return None
+ finally:
+ con.close()
def get_user_application_statuses(user_id: str) -> dict[tuple[str, str], str]:
@@ -265,7 +325,20 @@ def list_applications(user_id: str, status: str | None = None) -> list[dict]:
return []
-def update_status(app_id: str, status: str, notes: str | None = None) -> bool:
+def find_application_id(user_id: str, company: str, job_title: str) -> str | None:
+ """Id of this user's existing application for (company, title) — the dedup key — if any."""
+ con = sqlite3.connect(_DB_PATH)
+ row = con.execute(
+ "SELECT id FROM applications WHERE user_id=? AND lower(company)=lower(?) AND lower(job_title)=lower(?)",
+ (user_id, company or "", job_title or ""),
+ ).fetchone()
+ con.close()
+ return row[0] if row else None
+
+
+def update_status(app_id: str, status: str, notes: str | None = None, *, user_id: str) -> bool:
+ """Update one of `user_id`'s applications. False if the status is invalid or no such row
+ belongs to that user (so ids from other accounts can't be modified)."""
if status not in VALID_STATUSES:
logger.warning("Tracker: invalid status %r", status)
return False
@@ -279,11 +352,11 @@ def update_status(app_id: str, status: str, notes: str | None = None) -> bool:
if notes is not None:
updates.append("notes=?")
params.append(notes)
- params.append(app_id)
- con.execute(f"UPDATE applications SET {', '.join(updates)} WHERE id=?", params)
+ params += [app_id, user_id]
+ cur = con.execute(f"UPDATE applications SET {', '.join(updates)} WHERE id=? AND user_id=?", params)
con.commit()
con.close()
- return True
+ return cur.rowcount == 1
except Exception as exc:
logger.error("Tracker update_status failed: %s", exc)
return False
diff --git a/api/pytest.ini b/api/pytest.ini
new file mode 100644
index 0000000..6ad73c6
--- /dev/null
+++ b/api/pytest.ini
@@ -0,0 +1,10 @@
+[pytest]
+testpaths = tests
+# Lets a bare `pytest` (as CI runs it) import `app`, not just `python -m pytest`.
+pythonpath = .
+markers =
+ browser: drives a real headless Chromium through browser-use (slow; needs `playwright install chromium`)
+# The default run is the fast suite; `pytest -m browser` runs the browser tests instead.
+addopts = -m "not browser" -p no:cacheprovider
+filterwarnings =
+ ignore::DeprecationWarning
diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt
new file mode 100644
index 0000000..b96f05e
--- /dev/null
+++ b/api/requirements-dev.txt
@@ -0,0 +1,2 @@
+pytest>=8.0
+psutil>=5.9 # browser tests assert no Chromium processes are left behind
diff --git a/api/requirements.txt b/api/requirements.txt
index 4686e37..f7703ac 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -9,13 +9,15 @@ numpy>=1.26
bcrypt
passlib
python-jose[cryptography]
+cryptography>=42 # Fernet encryption of stored job-site passwords (app/crypto.py)
playwright>=1.45
-browser-use>=0.1
+# Pinned: browser_agent.py targets the 0.11 CDP API (its own Chat* LLM wrappers, actor Page,
+# keep_alive sessions). Minor releases have broken this before — upgrade deliberately.
+browser-use==0.11.13
+pydantic-settings>=2.0 # imported by browser_use/config.py on import, but missing from its declared deps
openai>=1.30 # OpenAI-compatible client (used for both Ollama + Groq)
-langchain-ollama>=0.2 # Ollama LangChain integration (browser agent)
-# Keep groq + langchain-groq if switching back via LLM_BACKEND=groq
-groq>=0.9
-langchain-groq>=0.2
+anthropic>=0.72.1 # Native Anthropic SDK (LLM_BACKEND=claude); floor set by browser-use 0.11.13
+groq>=0.30 # browser-use's ChatGroq (LLM_BACKEND=groq); floor set by browser-use 0.11.13
fpdf2>=2.7.9 # fallback PDF renderer
weasyprint>=62 # HTML/CSS -> PDF for structured CV / cover-letter templates
jinja2>=3.1 # HTML template rendering for documents
diff --git a/api/tests/browser/browser_support.py b/api/tests/browser/browser_support.py
new file mode 100644
index 0000000..7f5349d
--- /dev/null
+++ b/api/tests/browser/browser_support.py
@@ -0,0 +1,134 @@
+"""Fixtures and scripted LLMs for the real-browser tests (imported by test_browser_agent.py)."""
+import asyncio
+import functools
+import http.server
+import re
+import threading
+from pathlib import Path
+
+import psutil
+import pytest
+from browser_use.llm.views import ChatInvokeCompletion
+
+SITE = Path(__file__).parent / "site"
+
+
+@pytest.fixture(scope="module")
+def site():
+ """Serve tests/browser/site on 127.0.0.1; yields the base URL."""
+ handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(SITE))
+ handler.log_message = lambda *a, **k: None
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ yield f"http://127.0.0.1:{server.server_address[1]}"
+ server.shutdown()
+
+
+@pytest.fixture
+def ba(monkeypatch):
+ """app.browser_agent with IP blocking relaxed — the local test site is a raw IP."""
+ import app.browser_agent as module
+ monkeypatch.setattr(module, "_BLOCK_IP_ADDRESSES", False)
+ return module
+
+
+def chromium_processes() -> list:
+ found = []
+ for p in psutil.Process().children(recursive=True):
+ try:
+ name = p.name().lower()
+ if "chrom" in name or "headless_shell" in name:
+ found.append(p)
+ except psutil.Error:
+ pass
+ return found
+
+
+def assert_no_chromium_left(timeout: float = 10.0):
+ """Chromium shuts down asynchronously after kill(); wait for it rather than sleeping blind."""
+ async def wait():
+ for _ in range(int(timeout / 0.25)):
+ if not chromium_processes():
+ return
+ await asyncio.sleep(0.25)
+ asyncio.run(wait())
+ assert chromium_processes() == [], "Chromium processes left running"
+
+
+def _with_defaults(model_cls, data: dict):
+ for name, field in model_cls.model_fields.items():
+ if name not in data and field.is_required():
+ ann = field.annotation
+ data[name] = ([] if getattr(ann, "__origin__", None) is list
+ else "" if ann is str else False if ann is bool else None)
+ return model_cls.model_validate(data)
+
+
+class ScriptedLLM:
+ """
+ browser-use BaseChatModel stand-in. The fill agent and the submit agent each get one `done`
+ action (success configurable); other structured calls get schema defaults. Every message
+ text the model is sent is recorded in `.seen`.
+ """
+ model = "scripted"
+ _verified_api_keys = True
+ provider = property(lambda self: "scripted")
+ name = property(lambda self: "scripted")
+ model_name = property(lambda self: "scripted")
+
+ def __init__(self, fill_success: bool = True, submit_success: bool = True):
+ self.fill_success, self.submit_success = fill_success, submit_success
+ self.tasks: list[str] = []
+ self.seen: list[str] = []
+
+ @staticmethod
+ def _text(messages) -> str:
+ return "\n".join(str(getattr(m, "text", "") or getattr(m, "content", "")) for m in messages)
+
+ def next_action(self, text: str) -> list[dict]:
+ is_submit = "candidate has reviewed" in text
+ self.tasks.append("submit" if is_submit else "fill")
+ ok = self.submit_success if is_submit else self.fill_success
+ return [{"done": {"text": "submit agent ran" if is_submit else "Filled every field", "success": ok}}]
+
+ async def ainvoke(self, messages, output_format=None, **kwargs):
+ text = self._text(messages)
+ self.seen.append(text)
+ if output_format is None:
+ return ChatInvokeCompletion(completion="ok", usage=None)
+ if "action" not in output_format.model_fields:
+ return ChatInvokeCompletion(completion=_with_defaults(output_format, {}), usage=None)
+ return ChatInvokeCompletion(completion=_with_defaults(output_format, {"action": self.next_action(text)}),
+ usage=None)
+
+
+class SecretTypingLLM(ScriptedLLM):
+ """Types the password via its placeholder, reads the form back with JS (as a real model did), then finishes."""
+
+ def __init__(self):
+ super().__init__()
+ self.typed = self.read_back = False
+
+ def next_action(self, text: str) -> list[dict]:
+ field = re.search(r"\[(\d+)\]<input[^\n]*?(?:type=password|id=pw|name=password)", text)
+ if field and not self.typed:
+ self.typed = True
+ return [{"input": {"index": int(field.group(1)), "text": "<secret>job_password</secret>"}}]
+ if self.typed and not self.read_back:
+ self.read_back = True
+ return [{"evaluate": {"code": "(function(){return document.getElementById('pw').value})()"}}]
+ return [{"done": {"text": "ready for review", "success": True}}]
+
+
+async def run_apply_flow(ba, url: str, commands: list[str], **kwargs) -> list[dict]:
+ """Drive apply_with_browser, answering each interactive prompt with the next command."""
+ queue: asyncio.Queue = asyncio.Queue()
+ events = []
+ async for event in ba.apply_with_browser(url, {"core_info": {"first_name": "Ada"}}, "Dear Hiring Manager",
+ queue, kwargs.pop("frame_queue", None), **kwargs):
+ events.append(event)
+ if event.get("interactive") and commands:
+ await queue.put(commands.pop(0))
+ if event.get("done"):
+ break
+ return events
diff --git a/api/tests/browser/site/form.html b/api/tests/browser/site/form.html
new file mode 100644
index 0000000..c46c871
--- /dev/null
+++ b/api/tests/browser/site/form.html
@@ -0,0 +1,15 @@
+<!doctype html>
+<html><head><meta charset="utf-8"><title>Test Application Form
+
+
+