diff --git a/.gitignore b/.gitignore index 15546e4..e837568 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ frontend/playwright-report/ # Model evaluation output (api/evals) /api/evals/last_run.json /api/evals/last_doc_run.json +/api/evals/last_answers_run.json diff --git a/README.md b/README.md index b04bb5e..e835590 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ Built with FastAPI, Next.js, [browser-use](https://github.com/browser-use/browse 6. **Per-job analysis** — fit score (LLM semantic match) and AI risk for a single role. 7. **Tailored CV + Cover Letter** — structured, ATS-aware documents with a live in-app preview and PDF export (HTML/CSS → WeasyPrint). 8. **Browser Application** — a headless Chrome agent fills the application form and **stops before the final submit**. You review it in a live view (and can take over with clicks/typing), then `submit`, `done` (you submitted it yourself) or `skip`. -9. **Skill Roadmap** — 6 AI-resilient skill-development recommendations for your industry and target role. -10. **Application Tracker** — every application through a status lifecycle (Saved → Pending Review → Evaluated → Applied → Responded → Interview → Offer → Rejected / Discarded). +9. **Application questions** — every screening question on a form is answered from your answer bank, then your persona; written ones ("Why us?", "A time when…") are drafted for the job and fact-checked. A fact nothing you've given states is never guessed: it's left blank, and what you answer in the review is saved for the next form. +10. **Skill Roadmap** — 6 AI-resilient skill-development recommendations for your industry and target role. +11. **Application Tracker** — every application through a status lifecycle (Saved → Pending Review → Evaluated → Applied → Responded → Interview → Offer → Rejected / Discarded). --- @@ -158,6 +159,9 @@ Long-running endpoints stream **Server-Sent Events (SSE)**. Protected endpoints | `POST` | `/documents/pdf` | Render a structured doc (or legacy text) to PDF | | `POST` | `/documents/preview` | Render a structured doc to HTML for the live preview | | `GET` | `/documents/{doc_id}/download` | Download a generated PDF | +| `GET` / `POST` | `/answers` | Your answer bank / save an answer (`kind` fact or written; kept for one employer only when it names it) | +| `PUT` / `DELETE` | `/answers/{id}` | Change / delete a saved answer | +| `POST` | `/answers/draft` | Answer a job's application questions as the agent would (saved, profile, drafted or needs_you); nothing is saved | | `GET` | `/health` | Liveness check | --- @@ -196,6 +200,7 @@ Labels: **High ≥ 65 %**, **Medium 35–65 %**, **Low < 35 %**. **Job fit** (`s - **No telemetry** — browser-use's product telemetry (PostHog) and cloud sync default to on and would send each run's task prompt, actions and visited URLs to browser-use; `browser_agent.py` defaults `ANONYMIZED_TELEMETRY` / `BROWSER_USE_CLOUD_SYNC` to `false` and `docker-compose.yml` sets both. - **Network limits** — the agent's browser can't open raw-IP URLs, `localhost` or the internal service hostnames. - **CV upload** — the tailored CV PDF if you generated one, otherwise your uploaded original. +- **Questions** — the agent isn't given facts to answer from. It sends each page's questions to an `answer_questions` action (`answers.py`), which answers from your saved answers, then your persona, and drafts written ones, and says where each answer came from. A model maps each question to the fact that answers it, but the value comes from the fact, and code checks the reading (a cited fact, one of the form's options, a yes/no fact's direction, a UK fact not used for another country). Contact details, salary and equality answers reach that model only as "this fact exists". Questions nothing answers are left blank and listed first in the review, which shows every answer and its source. --- @@ -257,6 +262,8 @@ cd api && set -a && . ../.env && set +a WINDRUSH_LLM_EVAL=1 pytest evals -s # -s prints the scores; per-item answers go to evals/last_run.json ``` +`evals/test_answers_eval.py` puts 40 real application questions to a test persona with an answer bank. It fails if a fact is answered outside its labelled answers (a guess, a wrong option, a UK fact used for the US), a draft keeps an unsupported claim, or a private value reaches the model. + **Frontend smoke tests** (`frontend/e2e/`) drive the built app in Chromium with the API mocked: ```bash diff --git a/api/app/agent.py b/api/app/agent.py index 14f088a..6b16275 100644 --- a/api/app/agent.py +++ b/api/app/agent.py @@ -1566,8 +1566,9 @@ async def run_apply( job_password: str = "", cv_path: str = "", cover_letter_path: str = "", + answerer=None, ) -> AsyncGenerator[str, None]: - """Apply phase: browser automation.""" + """Apply phase: browser automation. `answerer` answers the form's questions (see answers.py).""" persona = tracker.get_user_persona(user_id) skill_risks = skill_risks or [] @@ -1580,7 +1581,7 @@ async def run_apply( async with aclosing(apply_with_browser( job_url, persona, cover_letter, instruction_queue, frame_queue, job_email=job_email, job_password=job_password, cv_path=cv_path, - cover_letter_path=cover_letter_path, + cover_letter_path=cover_letter_path, answerer=answerer, )) as steps: async for step in steps: event_type = "browser_blocked" if step.get("blocked") else "browser_action" @@ -1592,6 +1593,8 @@ async def run_apply( } if step.get("notes"): event["notes"] = step["notes"] + if step.get("questions"): + event["questions"] = step["questions"] yield _sse(event_type, event) if step.get("done"): submitted = bool(step.get("submitted")) diff --git a/api/app/answers.py b/api/app/answers.py new file mode 100644 index 0000000..f85ffb2 --- /dev/null +++ b/api/app/answers.py @@ -0,0 +1,682 @@ +""" +Application questions: how Windrush answers what a job application form asks. + +answer_questions() answers each question from one of these places, and says which (`source`): + saved - the answer bank: what the user answered or approved on an earlier form (tracker.answer_bank); + profile - a fact in the persona: right to work, notice period, nationality, a degree, a role's dates…; + drafted - a written answer (why this role, a time you…, a project) drafted for this job from the + persona, its stories and saved written answers, and checked for claims the persona doesn't + back. Drafts are for the user to read before anything is sent; + needs_you - a fact that neither the bank nor the persona states. It is never guessed: the form leaves it + blank, and once the user answers it the answer is saved for the next form. + +A model reads the questions, since their wording varies endlessly, and says which known fact answers +each. Code then checks that reading before anything goes on a form: an answer must cite a fact, be one of +the form's options, keep a yes/no fact's polarity, and not carry a UK fact to another country. Contact +details, salary and equality-monitoring answers never reach the model: it sees only that the fact exists, +and the value is filled in from the persona. +""" +import asyncio +import json +import logging +import re +from dataclasses import dataclass +from datetime import date, datetime + +from . import agent, doc_check, tracker +from .models import is_present_end_date + +logger = logging.getLogger("windrush.answers") + +SOURCES = ("saved", "profile", "drafted", "needs_you") + +_MAX_QUESTIONS = 40 # one form page rarely has more; a call with more is cut, not refused +_MAX_OPTIONS = 300 # a nationality dropdown lists every country +_DRAFTS_AT_ONCE = 4 + + +# ── Questions, normalised ────────────────────────────────────────────────────────────────────── + +_KEY_NOISE = re.compile(r"\((?:required|optional)\)|\*", re.I) +# A dropdown's prompt, not a choice: "Select...", "Please choose", "--". +_PLACEHOLDER = re.compile(r"^\W*(?:please\s+)?(?:select|choose|pick)\b|^\W*$", re.I) + + +def question_key(question: str) -> str: + """The question as matched against saved ones: lower case, without punctuation or 'required' marks.""" + text = _KEY_NOISE.sub(" ", str(question or "").lower()) + return " ".join(re.findall(r"[a-z0-9]+", text)) + + +def _norm(text: str) -> str: + return " ".join(re.findall(r"[a-z0-9]+", str(text or "").lower())) + + +def mentions(name: str, text: str) -> bool: + """Whether `text` names `name` (a company), as whole words.""" + name = _norm(name) + return bool(name) and re.search(rf"(? list[dict]: + out, seen = [], set() + for raw in (questions or [])[:_MAX_QUESTIONS]: + if isinstance(raw, str): + raw = {"question": raw} + if not isinstance(raw, dict): + continue + text = " ".join(str(raw.get("question") or "").split())[:500] + key = question_key(text) + if not key or key in seen: + continue + seen.add(key) + options = [" ".join(str(o).split())[:200] for o in raw.get("options") or [] + if not _PLACEHOLDER.search(str(o))][:_MAX_OPTIONS] + limit = raw.get("max_length") + out.append({"question": text, "options": options, + "max_length": int(limit) if isinstance(limit, (int, float)) and limit > 0 else None}) + return out + + +# ── What Windrush knows: facts from the persona and the answer bank ──────────────────────────── + +@dataclass(frozen=True) +class Fact: + id: str + label: str # what the fact is about: "Right to work / work authorisation in the UK" + value: str # its answer as a form takes it: "Yes", "2 weeks", "British" + private: bool = False # the value never goes into a model prompt + source: str = "profile" # "saved" for answer-bank entries + uk: bool = False # only says something about the UK (right to work, sponsorship) + yes_no: bool = False + bank_id: str = "" + + +# (section, field, label, about the UK only). Unanswered (None) ones are left out entirely, so nothing +# is ever stated that the user hasn't told us. +_YES_NO_FACTS = ( + ("core_info", "right_to_work_uk", "Right to work / work authorisation in the UK", True), + ("core_info", "require_sponsorship", "Require visa sponsorship to work in the UK, now or in the future", True), + ("core_info", "has_government_ties", "Has government ties", False), + ("core_info", "holds_other_nationality", "Currently holds another (dual) nationality or citizenship", False), + ("core_info", "held_other_nationality", "Previously held another nationality or citizenship", False), + ("preferences", "relocation_willingness", "Willing to relocate", False), + ("preferences", "can_work_in_person", "Can work in-person", False), + ("preferences", "can_start_immediately", "Can start immediately", False), + ("preferences", "has_reliable_transportation", "Has reliable transportation", False), + ("preferences", "needs_accommodations", "Needs adjustments or accommodations", False), +) + +# (field, label, private) +_CORE_TEXT_FACTS = ( + ("first_name", "First name", False), + ("last_name", "Last name", False), + ("preferred_name", "Preferred name", False), + ("email", "Email address", True), + ("phone", "Phone number", True), + ("address_line_1", "Street address", True), + ("postcode", "Postcode", True), + ("city", "Town or city where they live", False), + ("country", "Country where they live", False), + ("linkedin", "LinkedIn profile URL", False), + ("github", "GitHub profile URL", False), + ("twitter", "Twitter / X profile", False), + ("portfolio", "Portfolio URL", False), + ("website", "Personal website URL", False), + ("visa_status", "Visa / immigration status", False), + ("visa_type", "Visa type", False), + ("nationality", "Nationality", False), + ("security_clearance", "Security clearance held", False), + ("dob", "Date of birth", True), +) + +_DISABILITY = {"yes": "Yes", "no": "No", "prefer_not_to_say": "Prefer not to say"} +_DECLINE = "Prefer not to say" + +# A year (19xx/20xx) in a role's end date: the role has actually ended — unless a present word is +# in it too, as when "Mid-2019-Present" splits at its first hyphen or a CV lists stints ("Nov 2012, +# Dec 2012 – Present"). +_END_YEAR = re.compile(r"(? bool: + end = str(end_date or "") + return bool(_END_YEAR.search(end)) and not _PRESENT_WORD.search(end) + + +def _birth_date(raw: str) -> date | None: + for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%d-%m-%Y", "%d.%m.%Y", "%d %B %Y", "%d %b %Y", "%B %d, %Y"): + try: + return datetime.strptime(raw.strip(), fmt).date() + except ValueError: + continue + return None + + +def _age(born: date, today: date) -> int: + return today.year - born.year - ((today.month, today.day) < (born.month, born.day)) + + +def _dates(entry: dict, start: str, end: str, current: str) -> str: + finish = "Present" if entry.get(current) else (entry.get(end) or "") + span = " – ".join(p for p in (entry.get(start) or "", finish) if p) + return f" ({span})" if span else "" + + +def profile_facts(persona: dict, today: date | None = None) -> list[Fact]: + """Every fact the persona states that a form might ask for, as the answer a form takes.""" + core = persona.get("core_info") or {} + prefs = persona.get("preferences") or {} + screening = persona.get("screening") or {} + diversity = persona.get("diversity") or {} + sections = {"core_info": core, "preferences": prefs} + facts: list[Fact] = [] + + def add(fact_id, label, value, **kw): + value = " ".join(str(value or "").split()) + if value: + facts.append(Fact(fact_id, label, value, **kw)) + + add("full_name", "Full name", " ".join(p for p in (core.get("first_name"), core.get("last_name")) if p)) + for field, label, private in _CORE_TEXT_FACTS: + add(field, label, core.get(field), private=private) + for section, field, label, uk in _YES_NO_FACTS: + value = sections[section].get(field) + if isinstance(value, bool): + facts.append(Fact(field, label, "Yes" if value else "No", uk=uk, yes_no=True)) + # The details only follow a "Yes": text left over from an earlier answer says nothing. + if core.get("holds_other_nationality") is True: + add("other_nationalities", "Other current nationalities / citizenships", core.get("other_nationalities")) + if core.get("held_other_nationality") is True: + add("previous_nationalities", "Previously held nationalities / citizenships", core.get("previous_nationalities")) + born = _birth_date(str(core.get("dob") or "")) if core.get("dob") else None + if born: + facts.append(Fact("over_18", "Is 18 or older", "Yes" if _age(born, today or date.today()) >= 18 else "No", + yes_no=True)) + + if prefs.get("remote_preference"): + add("remote_preference", "Preferred work setting (remote / hybrid / on-site)", prefs["remote_preference"]) + add("preferred_locations", "Preferred work locations", + ", ".join(loc for loc in prefs.get("preferred_locations") or [] if loc)) + add("notice_period", "Notice period", screening.get("notice_period_canonical") or prefs.get("notice_period")) + # Salary is the candidate's call (a figure, a range, "Negotiable"): with none in the profile it is + # left for them rather than answered for them. + add("salary_annual", "Salary expectations (Annual)", screening.get("salary_canonical") or prefs.get("min_salary"), + private=True) + add("salary_hourly", "Salary expectations (Hourly)", prefs.get("expected_hourly_rate"), private=True) + + # Declining to answer is always a true statement, so it's the fallback for equality monitoring. + add("gender", "Gender (equality monitoring)", diversity.get("gender") or _DECLINE, private=True) + add("ethnicity", "Ethnicity (equality monitoring)", diversity.get("ethnicity") or _DECLINE, private=True) + add("disability", "Disability (equality monitoring)", + _DISABILITY.get(diversity.get("disability_status") or "", _DECLINE), private=True) + add("veteran_status", "Veteran status (equality monitoring)", diversity.get("veteran_status") or _DECLINE, + private=True) + add("sexual_orientation", "Sexual orientation (equality monitoring)", + diversity.get("sexual_orientation") or _DECLINE, private=True) + + # Parsed CVs often leave is_current off and put "Present" (or nothing) in end_date, so "No" needs + # every role to have really ended — an end date with a year in it; anything less certain is left + # for the user (an end date we don't recognise may still mean "present"). + history = [job for job in persona.get("history") or [] if isinstance(job, dict)] + current = [job for job in history if job.get("is_current") or is_present_end_date(job.get("end_date"))] + if current: + facts.append(Fact("currently_employed", "Are you currently employed", "Yes", yes_no=True)) + add("current_employer", "Current employer", current[0].get("employer")) + add("current_title", "Current job title", current[0].get("title")) + elif history and all(_has_ended(job.get("end_date")) for job in history): + facts.append(Fact("currently_employed", "Are you currently employed", "No", yes_no=True)) + for i, job in enumerate(history): + add(f"role_{i + 1}", "Work history", f"{job.get('title', '')} at {job.get('employer', '')}" + + _dates(job, "start_date", "end_date", "is_current")) + for i, edu in enumerate(e for e in persona.get("education") or [] if isinstance(e, dict)): + grade = f", {edu['grade']}" if edu.get("grade") else "" + add(f"education_{i + 1}", "Education", f"{edu.get('degree', '')} at {edu.get('institution', '')}" + + _dates(edu, "start_date", "end_date", "is_currently_enrolled") + grade) + skills = [s for group in persona.get("skills") or [] for s in group.get("skills") or []] + add("skills", "Skills", ", ".join(dict.fromkeys(skills))) + certs = [f"{c.get('name', '')} ({c.get('issuing_organization', '')})".replace(" ()", "") + for c in persona.get("certifications") or [] if c.get("name")] + add("certifications", "Certifications held", "; ".join(certs)) + facts.append(Fact("heard_about", "How you heard about this job", + "Online job search (pick the closest option, e.g. 'Company website' or 'Internet search')")) + return facts + + +def bank_facts(bank: list[dict], company: str = "") -> list[Fact]: + """Saved fact answers this form may reuse: general ones, and those saved for this employer.""" + facts = [] + for n, entry in enumerate(e for e in bank if e.get("kind") == "fact" and _bank_applies(e, company)): + about = f" (for {entry['company']})" if entry.get("company") else "" + facts.append(Fact(f"saved_{n + 1}", f"Saved answer to “{entry['question']}”{about}", entry["answer"], + source="saved", yes_no=_polarity(entry["answer"]) is not None + and len(_norm(entry["answer"]).split()) == 1, bank_id=entry["id"])) + return facts + + +def _bank_applies(entry: dict, company: str) -> bool: + return not entry.get("company") or _norm(entry["company"]) == _norm(company) + + +# ── Reading answers against a question ───────────────────────────────────────────────────────── + +_YES = {"yes", "y", "true"} +_NO = {"no", "n", "false"} + + +def _polarity(text: str) -> bool | None: + """True for an answer that starts 'Yes', False for 'No', None for anything else.""" + words = _norm(text).split() + if not words: + return None + return True if words[0] in _YES else False if words[0] in _NO else None + + +_NEGATED = re.compile(r"\b(?:not|without|never|unable|no longer|neither|nor)\b|n['’]t\b", re.I) +_DECLINE_RE = re.compile(r"prefer not|rather not|decline|don['’]?t wish|do not wish|not to (?:say|disclose|answer)", re.I) + + +def _in_options(answer: str, options: list[str]) -> str | None: + """The option `answer` names, compared without case or punctuation.""" + wanted = _norm(answer) + return next((o for o in options if _norm(o) == wanted), None) if wanted else None + + +def pick_option(value: str, options: list[str]) -> str | None: + """The one option that states `value`, or None when no option or more than one does.""" + if not options: + return value + v = _norm(value) + if not v: + return None + normed = [(_norm(o), o) for o in options] + exact = [o for n, o in normed if n == v] + if exact: + return exact[0] + pol = _polarity(v) + if pol is not None and len(v.split()) == 1: + same = [o for n, o in normed if _polarity(n) is pol] + return same[0] if len(same) == 1 else None + if _DECLINE_RE.search(value): + declines = [o for n, o in normed if _DECLINE_RE.search(o)] + return declines[0] if len(declines) == 1 else None + for match in (lambda n: n.startswith(v + " ") or v.startswith(n + " "), + lambda n: f" {v} " in f" {n} " or f" {n} " in f" {v} "): + found = [o for n, o in normed if n and match(n)] + if len(found) == 1: + return found[0] + if found: + return None + return None + + +_UK_RE = re.compile(r"\b(?:uk|u\.k\.|united kingdom|great britain|britain|british|england|english|scotland|" + r"scottish|wales|welsh|northern ireland|gb)\b", re.I) +# Bare "us" counts only in capitals ("the US"): "tell us", "join us". +_US_RE = re.compile(r"\bUS\b") +_ELSEWHERE_RE = re.compile( + r"\b(?:u\.s\.a?\.?|usa|united states|america|american|canada|canadian|ireland|irish|republic of ireland|" + r"eu|e\.u\.|european union|eea|europe|european|schengen|australia|australian|new zealand|india|indian|" + r"germany|german|france|french|netherlands|dutch|spain|spanish|italy|italian|switzerland|swiss|singapore|" + r"hong kong|japan|china|uae|united arab emirates|dubai|israel|poland|sweden|denmark|norway|finland|belgium|" + r"austria|portugal|brazil|mexico|south africa|nigeria|kenya|h-?1b|green card)\b", re.I) + + +# Job locations outside the UK that name no country: big tech cities, and US "City, ST" addresses. +_ELSEWHERE_PLACE_RE = re.compile( + r"(?i:\b(?:new york|nyc|san francisco|bay area|seattle|boston|austin|chicago|los angeles|denver|toronto|" + r"vancouver|montreal|berlin|munich|paris|amsterdam|dublin|madrid|barcelona|lisbon|stockholm|copenhagen|" + r"zurich|warsaw|sydney|melbourne|bangalore|bengaluru|tokyo|tel aviv)\b)|,\s*[A-Z]{2}\b(?!\s*\w)") + + +def about_uk(question: str, job_location: str = "") -> bool: + """Whether a UK fact can answer this question: it names the UK, or names no country and the job isn't + elsewhere. A job with no location, or a remote one, is taken to be in the UK, where Windrush searches.""" + def elsewhere(text): + return bool(_ELSEWHERE_RE.search(text) or _US_RE.search(text)) + if _UK_RE.search(question): + return True + if elsewhere(question): + return False + location = job_location or "" + if _UK_RE.search(location): + return True + return not (elsewhere(location) or _ELSEWHERE_PLACE_RE.search(location)) + + +_STOP = {"a", "an", "the", "of", "in", "at", "and", "or", "to", "for", "with", "on", "i", "am", "have", "has", + "my", "is", "be", "s", "it", "as"} + + +def _supported(answer: str, facts: list[Fact]) -> bool: + """A free-text answer the model wrote from shown facts may only use their words.""" + source = set(re.findall(r"[a-z0-9]+", " ".join(f"{f.label} {f.value}" for f in facts).lower())) + return set(re.findall(r"[a-z0-9]+", answer.lower())) - _STOP <= source + + +def resolve_fact(question: dict, reading: dict, facts: dict[str, Fact], job_location: str = "") -> tuple[str, list[Fact]] | None: + """ + The answer to a fact question that a model's `reading` ({fact_ids, answer}) supports, with the facts + it rests on — or None when it can't be trusted, which leaves the question for the user. + """ + cited = [facts[i] for i in reading.get("fact_ids") or [] if isinstance(i, str) and i in facts] + if not cited: + return None + text, options = question["question"], question["options"] + if any(f.uk for f in cited) and not about_uk(text, job_location): + return None + said = str(reading.get("answer") or "").strip() + first = cited[0] + + if len(cited) == 1 and first.private: + # The model never saw the value, so it can only have said which fact applies. + answer = pick_option(first.value, options) + return (answer, cited) if answer else None + + if options: + answer = _in_options(said, options) + if answer is None and len(cited) == 1: + answer = pick_option(first.value, options) + if answer is None: + return None + elif said: + answer = said + if _polarity(said) is None and not _supported(said, cited): + if len(cited) != 1: + return None + answer = first.value + else: + if len(cited) != 1: + return None + answer = first.value + + # A yes/no fact keeps its polarity unless the question turns it round ("…without sponsorship?"). + if len(cited) == 1 and first.yes_no and not _NEGATED.search(text): + if _polarity(answer) is not None and _polarity(answer) != _polarity(first.value): + return None + return answer, cited + + +# ── The model's reading of the questions ─────────────────────────────────────────────────────── + +_READ_SYSTEM = ( + "You read the questions on a job application form for one candidate and match each to the facts Windrush " + "holds about them. You never answer from general knowledge, from what is typical, or by guessing; only by " + "a plain reading of the facts.\n" + "For each question decide its kind:\n" + "- \"written\": it asks the candidate to write in their own words: why they want the role or to join the " + "company, their interest, experience, skills or achievements, a time they did something, a project, " + "strengths, goals, or anything else they'd like to add.\n" + "- \"fact\": anything else: eligibility, right to work, visas, status, availability, notice, salary, " + "preferences, contact details, education, work history, a number, a choice between options about them, " + "equality monitoring, consent or a declaration.\n" + "For a \"fact\" give fact_ids, the ids of the facts that state the answer, and answer, the answer they give " + "in the form's terms: when options are listed, exactly one option copied word for word. If no fact states " + "it, fact_ids is [] and answer is \"\". Rules:\n" + "- A fact about one country never answers a question about another. The job's location is given for " + "questions about 'this country' or 'the country where this role is based'.\n" + "- A fact marked (private) has its value hidden: cite its id when it is the fact asked for, and leave " + "answer \"\"; Windrush fills in the value.\n" + "- Don't compute or estimate: years of experience, counts, grades or dates are answered only by a fact " + "that states them.\n" + "- Read the question's direction: 'Will you require sponsorship?' and 'Can you work without " + "sponsorship?' take opposite answers from the same fact.\n" + "- A declaration or consent (a privacy notice, that the information is accurate, background or reference " + "checks, criminal convictions, conflicts of interest) is answered only by a saved answer to it.\n" + 'Return ONLY JSON: {"answers": [{"n": int, "kind": "fact" | "written", "fact_ids": [str], "answer": str}]} ' + "with one entry per question." +) + + +def _fact_lines(facts: list[Fact]) -> str: + return "\n".join(f"- {f.id}: {f.label} (private)" if f.private else f"- {f.id}: {f.label}: {f.value}" + for f in facts) + + +def _question_lines(questions: list[dict]) -> str: + lines = [] + for n, q in enumerate(questions, 1): + line = f"{n}. {q['question']}" + if q["options"]: + line += "\n options: " + " | ".join(q["options"]) + lines.append(line) + return "\n".join(lines) + + +async def _read(questions: list[dict], facts: list[Fact], job: dict) -> dict[int, dict]: + """The model's reading of each question, by its 1-based position; {} if the call failed.""" + data = await agent._llm_json( + system=_READ_SYSTEM, + user=(f"JOB: {job.get('title', '')} at {job.get('company', '')}, located in: " + f"{job.get('location') or 'not stated'}\n\nFACTS:\n{_fact_lines(facts)}\n\n" + f"QUESTIONS:\n{_question_lines(questions)}"), + max_tokens=4000, label="read_questions") + readings = {} + for item in (data or {}).get("answers") or []: + if isinstance(item, dict) and isinstance(item.get("n"), int) and item.get("kind") in ("fact", "written"): + ids = item.get("fact_ids") + readings[item["n"]] = {"kind": item["kind"], "fact_ids": ids if isinstance(ids, list) else [], + "answer": str(item.get("answer") or "")} + return readings + + +# ── Written answers ──────────────────────────────────────────────────────────────────────────── + +_WORD_LIMIT = re.compile(r"(\d{2,4})\s*(?:-\s*\d{2,4}\s*)?words?\b", re.I) +_CHAR_LIMIT = re.compile(r"(\d{2,5})\s*(?:characters|chars)\b", re.I) + + +def _limits(question: dict) -> tuple[int | None, int | None]: + """(max words, max characters) the question or its text box sets. A box limit equal to the question's word + limit is that word limit read as characters (the fill agent did so in a live run), so it's ignored.""" + words = _WORD_LIMIT.search(question["question"]) + max_words = int(words.group(1)) if words else None + chars = _CHAR_LIMIT.search(question["question"]) + box = question.get("max_length") + if box is not None and box == max_words: + box = None + max_chars = min(filter(None, [int(chars.group(1)) if chars else None, box]), default=None) + return max_words, max_chars + + +def _length_rule(max_words: int | None, max_chars: int | None) -> str: + if max_chars: + return f"at most {max_chars} characters including spaces (aim for about {int(max_chars * 0.85)})" + if max_words: + return f"at most {max_words} words (aim for about {max(20, int(max_words * 0.85))})" + return "80-150 words" + + +def _fit(text: str, max_words: int | None, max_chars: int | None) -> str: + """Cut an answer that runs over its limit at the last whole sentence that fits.""" + def fits(t): + return (not max_chars or len(t) <= max_chars) and (not max_words or len(t.split()) <= max_words) + if fits(text): + return text + sentences = re.split(r"(?<=[.!?])\s+", text) + while len(sentences) > 1 and not fits(" ".join(sentences)): + sentences.pop() + kept = " ".join(sentences) + if not fits(kept): + kept = kept[:max_chars] if max_chars else " ".join(kept.split()[:max_words]) + return kept.strip() + + +def _similar(question: str, entries: list[dict], limit: int = 3) -> list[dict]: + want = set(question_key(question).split()) - _STOP + scored = [] + for e in entries: + have = set(question_key(e["question"]).split()) - _STOP + overlap = len(want & have) / max(1, len(want | have)) + if overlap >= 0.2: + scored.append((overlap, e)) + return [e for _, e in sorted(scored, key=lambda s: -s[0])[:limit]] + + +_WRITE_SYSTEM = ( + "You write one candidate's answer to one question on a job application form: first person, plain UK " + "English, as the candidate would write it. Use ONLY facts in the candidate data, their saved answers and " + "their cover letter: never invent a number, technology, employer, project, result or experience, and " + "never claim more than the data says. Answer the question in the first sentence, then back it with one or " + "two specific pieces of evidence (a named project, role or story and its real result). For a behavioural " + "question ('a time when…'), tell one story from the story bank in situation, task, action, result order, " + "without those labels. For why this role or company, tie what the advert says this team " + "does to the candidate's own experience, so that no other company's name could replace this one. Where a " + "saved answer fits, keep its substance and adapt it to this job. Length: {length}. No greeting, sign-off, " + "headings or bullet points. Don't use these words: {banned}. " + 'Return ONLY JSON: {{"answer": str}}' +) + + +async def _write(question: dict, persona: dict, job: dict, cover_letter: str, saved: list[dict], + fix: str = "", draft: str = "") -> str: + max_words, max_chars = _limits(question) + user = (f"QUESTION: {question['question']}\n\n" + f"JOB: {job.get('title', '')} at {job.get('company', '')}\n" + f"ADVERT: {(job.get('description') or '')[:3000]}\n\n" + + ("SAVED ANSWERS TO SIMILAR QUESTIONS:\n" + + "\n".join(f"- {e['question']}: {e['answer']}" for e in saved) + "\n\n" if saved else "") + + (f"COVER LETTER FOR THIS JOB:\n{cover_letter[:3000]}\n\n" if cover_letter.strip() else "") + + f"CANDIDATE DATA (JSON): {json.dumps(agent._llm_persona(persona), ensure_ascii=False)}") + if fix: + user = f"YOUR DRAFT:\n{draft}\n\nFIX ONLY THESE ISSUES, keeping everything else:\n{fix}\n\n" + user + data = await agent._llm_json( + system=_WRITE_SYSTEM.format(length=_length_rule(max_words, max_chars), + banned=", ".join(doc_check.BANNED_WORDS)), + user=user, max_tokens=2000, label="revise_answer" if fix else "write_answer") + answer = (data or {}).get("answer") + return " ".join(answer.split()) if isinstance(answer, str) else "" + + +def _findings(answer: str, persona: dict, job: dict, extra_source: str) -> list[dict]: + doc = {"paragraphs": [answer]} + return (doc_check.check_facts("cover_letter", doc, persona, job, extra_source) + + doc_check.check_style("cover_letter", doc)) + + +async def draft_answer(question: dict, persona: dict, job: dict, cover_letter: str = "", + saved: list[dict] | None = None) -> tuple[str, list[str]]: + """A written answer for this job, and what the document check still flags in it. One revision runs + when the check flags something, and is kept only if it clears findings without adding any.""" + saved = saved or [] + extra = "\n".join([cover_letter] + [e["answer"] for e in saved]) + answer = await _write(question, persona, job, cover_letter, saved) + if not answer: + return "", [] + found = _findings(answer, persona, job, extra) + if found: + revised = await _write(question, persona, job, cover_letter, saved, + fix=agent._issue_lines(found), draft=answer) + if revised: + after = _findings(revised, persona, job, extra) + if agent._better(found, after): + answer, found = revised, after + max_words, max_chars = _limits(question) + return _fit(answer, max_words, max_chars), [f"{f['label']}: {f['message']}" for f in found] + + +# ── Answering a form's questions ─────────────────────────────────────────────────────────────── + +def _result(question: dict, kind: str, source: str, answer: str = "", because: str = "", + bank_id: str = "", flags: list[str] | None = None) -> dict: + return {"question": question["question"], "options": question["options"], "kind": kind, "source": source, + "answer": answer if source != "needs_you" else "", "because": because, "bank_id": bank_id, + "flags": flags or []} + + +def _saved_match(question: dict, bank: list[dict], company: str) -> dict | None: + """The saved answer to this very question, preferring one saved for this employer.""" + key = question_key(question["question"]) + matches = [e for e in bank if e.get("question_key") == key and _bank_applies(e, company)] + return max(matches, key=lambda e: bool(e.get("company")), default=None) + + +def _because(facts: list[Fact]) -> str: + def one(f): + if f.source == "saved": + return f.label + return f"Your profile: {f.label}" + ("" if f.private else f" = {f.value}") + return "; ".join(one(f) for f in facts) + + +async def answer_questions(questions, persona: dict, bank: list[dict], job: dict, cover_letter: str = "") -> list[dict]: + """ + Answers for a form's questions, in order: each {question, options, kind ('fact' | 'written'), source (one + of SOURCES), answer ('' when needs_you), because (what it rests on), bank_id, flags (what the document + check still flags in a draft)}. `questions` are {question, options?, max_length?} or plain strings. + """ + questions = _clean_questions(questions) + company = job.get("company", "") + results: list[dict | None] = [None] * len(questions) + + # The very question answered before: reused as saved, when it fits the form's options. + for i, q in enumerate(questions): + entry = _saved_match(q, bank, company) + if entry: + answer = pick_option(entry["answer"], q["options"]) if entry["kind"] == "fact" else entry["answer"] + if answer: + results[i] = _result(q, entry["kind"], "saved", answer, f"Saved answer to “{entry['question']}”", + entry["id"]) + + pending = [i for i, r in enumerate(results) if r is None] + if not pending: + return results + facts = profile_facts(persona) + bank_facts(bank, company) + by_id = {f.id: f for f in facts} + readings = await _read([questions[i] for i in pending], facts, job) + + written = [] + for n, i in enumerate(pending, 1): + q, reading = questions[i], readings.get(n) + if reading is None: + # Unread (the model call failed): nothing is stated for it. + results[i] = _result(q, "fact", "needs_you", because="Windrush couldn't read this question") + elif reading["kind"] == "written": + written.append(i) + else: + resolved = resolve_fact(q, reading, by_id, job.get("location", "")) + if resolved: + answer, rests_on = resolved + saved = next((f for f in rests_on if f.source == "saved"), None) + results[i] = _result(q, "fact", "saved" if saved else "profile", answer, _because(rests_on), + saved.bank_id if saved else "") + else: + results[i] = _result(q, "fact", "needs_you", because="Nothing you've saved says this") + + saved_written = [e for e in bank if e.get("kind") == "written" and _bank_applies(e, company)] + gate = asyncio.Semaphore(_DRAFTS_AT_ONCE) + + async def draft(i): + async with gate: + q = questions[i] + similar = _similar(q["question"], saved_written) + try: + answer, flags = await draft_answer(q, persona, job, cover_letter, similar) + except Exception as exc: + logger.warning("drafting an answer failed: %s", exc) + answer, flags = "", [] + if not answer: + return _result(q, "written", "needs_you", because="Windrush couldn't draft this one") + basis = (f"Drafted for this job from your persona and saved answers to “{similar[0]['question']}”" + if similar else "Drafted for this job from your persona") + return _result(q, "written", "drafted", answer, basis, similar[0]["id"] if similar else "", flags) + + for i, result in zip(written, await asyncio.gather(*(draft(i) for i in written))): + results[i] = result + return results + + +# ── The answer bank ──────────────────────────────────────────────────────────────────────────── + +def save(user_id: str, question: str, answer: str, kind: str = "fact", company: str = "") -> dict: + """Save (or replace) the user's answer to a question. It is kept for one employer only when the question + or the answer names that employer ("Why do you want to join Acme?"), so other forms never reuse it.""" + question = " ".join(str(question or "").split()) + answer = str(answer or "").strip() + if not question_key(question) or not answer: + raise ValueError("a question and an answer are both needed") + if kind not in tracker.ANSWER_KINDS: + raise ValueError(f"kind must be one of {tracker.ANSWER_KINDS}") + company = company.strip() if company and mentions(company, f"{question}\n{answer}") else "" + return tracker.save_answer(user_id, question, question_key(question), answer, kind, company) diff --git a/api/app/browser_agent.py b/api/app/browser_agent.py index 280bd31..3c76260 100644 --- a/api/app/browser_agent.py +++ b/api/app/browser_agent.py @@ -20,8 +20,9 @@ import unicodedata from typing import AsyncGenerator, Awaitable, Callable +from pydantic import BaseModel, Field + from . import azure_config -from .models import is_present_end_date # browser-use's product telemetry (PostHog) and cloud sync are ON unless these say otherwise: every # run would send the task prompt (the applicant's personal data), action history, visited URLs and @@ -358,6 +359,7 @@ async def _interactive_session( intro_reason: str, on_submit: Callable[[], Awaitable[tuple[bool, str]]], notes: str = "", + questions: list[dict] | None = None, ) -> AsyncGenerator[dict, None]: """ Hand control to the user until they finish. Commands (from BrowserView.tsx): @@ -366,7 +368,8 @@ async def _interactive_session( 'done' → the user submitted it themselves 'skip' / 'cancel' / 'abort' → abandon (not submitted) any other text → typed into the focused field - `notes` (the agent's own summary) rides on the first event only; the UI keeps it for the review. + `notes` (the agent's own summary) and `questions` (what answer_questions answered) ride on the first + event only; the UI keeps them for the review. """ def interactive(action: str, reason: str, screenshot: str) -> dict: return {"action": action, "screenshot": screenshot or None, "blocked": True, @@ -375,6 +378,8 @@ def interactive(action: str, reason: str, screenshot: str) -> dict: intro = interactive(intro_action, intro_reason, await _screenshot_b64(browser_session)) if notes: intro["notes"] = notes + if questions: + intro["questions"] = questions yield intro again = "Click the screenshot or type below. 'submit' sends it, 'done' if you submitted it yourself, 'skip' cancels." @@ -435,79 +440,6 @@ def interactive(action: str, reason: str, screenshot: str) -> dict: yield interactive(label, again, await _screenshot_b64(browser_session)) -# Yes/no facts stated on the candidate's behalf. Unanswered (None/missing) ones are left out of -# the prompt entirely, so the agent never asserts something the user hasn't told us. -_YES_NO_QUESTIONS = ( - ("core_info", "right_to_work_uk", "Right to work / work authorisation in the UK"), - ("core_info", "require_sponsorship", "Require visa sponsorship"), - ("core_info", "has_government_ties", "Has government ties"), - ("core_info", "holds_other_nationality", "Currently holds another (dual) nationality or citizenship"), - ("core_info", "held_other_nationality", "Previously held another nationality or citizenship"), - ("preferences", "relocation_willingness", "Willing to relocate"), - ("preferences", "can_work_in_person", "Can work in-person"), - ("preferences", "can_start_immediately", "Can start immediately"), - ("preferences", "has_reliable_transportation", "Has reliable transportation"), - ("preferences", "needs_accommodations", "Needs accommodations"), -) - -_DISABILITY_ANSWERS = {"yes": "Yes", "no": "No", "prefer_not_to_say": "Prefer not to say"} - -# A year (19xx/20xx) in a role's end date: the role has actually ended — unless a present word is -# in it too, as when "Mid-2019-Present" splits at its first hyphen or a CV lists stints ("Nov 2012, -# Dec 2012 – Present"). -_END_YEAR = re.compile(r"(? bool: - end = str(end_date or "") - return bool(_END_YEAR.search(end)) and not _PRESENT_WORD.search(end) - - -def _known_answers(persona: dict) -> list[tuple[str, str]]: - core = persona.get("core_info") or {} - prefs = persona.get("preferences") or {} - screening = persona.get("screening") or {} - diversity = persona.get("diversity") or {} - sections = {"core_info": core, "preferences": prefs} - - answers = [] - for section, field, question in _YES_NO_QUESTIONS: - value = sections[section].get(field) - if isinstance(value, bool): - answers.append((question, "Yes" if value else "No")) - - # Declining to answer is always a true statement, so it's the fallback for diversity questions. - answers += [ - ("Gender", diversity.get("gender") or "Prefer not to say"), - ("Ethnicity / diversity questions", diversity.get("ethnicity") or "Prefer not to say"), - ("Disability", _DISABILITY_ANSWERS.get(diversity.get("disability_status") or "", "Prefer not to say")), - ] - # Salary is the candidate's call (a figure, a range, "Negotiable"): with none in the profile it is - # left for them rather than answered for them. - annual = screening.get("salary_canonical") or prefs.get("min_salary") - if annual: - answers.append(("Salary expectations (Annual)", str(annual))) - if prefs.get("expected_hourly_rate"): - answers.append(("Salary expectations (Hourly)", str(prefs["expected_hourly_rate"]))) - if prefs.get("remote_preference"): - answers.append(("Remote/Hybrid preference", prefs["remote_preference"])) - locations = ", ".join(loc for loc in prefs.get("preferred_locations") or [] if loc) - if locations: - answers.append(("Preferred work locations", locations)) - # Parsed CVs often leave is_current off and put "Present" (or nothing) in end_date, so "No" needs - # every role to have really ended — an end date with a year in it; anything less certain is left - # for the user (an end date we don't recognise may still mean "present"). - history = [job for job in persona.get("history") or [] if isinstance(job, dict)] - if any(job.get("is_current") or is_present_end_date(job.get("end_date")) for job in history): - answers.append(("Are you currently employed", "Yes")) - elif history and all(_has_ended(job.get("end_date")) for job in history): - answers.append(("Are you currently employed", "No")) - answers.append(("How did you hear about us / about this role", - "Website / Internet search (pick the closest option, e.g. 'Company website' or 'Internet')")) - return answers - - def _build_task(job_url: str, persona: dict, cover_letter: str, job_email: str, job_password: str, cv_path: str, cover_letter_path: str = "") -> str: lines = [] @@ -520,8 +452,7 @@ def _build_task(job_url: str, persona: dict, cover_letter: str, lines.append(f"If asked to upload a CV/resume, upload the file at: {cv_path}") core = persona.get("core_info", {}) - prefs = persona.get("preferences", {}) - + profile_lines = [] full_name = f"{core.get('first_name', '')} {core.get('last_name', '')}".strip() if full_name: profile_lines.append(f"Full name: {full_name}") @@ -539,14 +470,8 @@ def _build_task(job_url: str, persona: dict, cover_letter: str, if core.get("github"): profile_lines.append(f"GitHub: {core['github']}") if core.get("twitter"): profile_lines.append(f"Twitter/X: {core['twitter']}") if core.get("portfolio"): profile_lines.append(f"Portfolio/Website: {core['portfolio']}") - if core.get("visa_status"): profile_lines.append(f"Visa Status: {core['visa_status']}") - if core.get("visa_type"): profile_lines.append(f"Visa Type: {core['visa_type']}") - if core.get("security_clearance"): profile_lines.append(f"Security Clearance: {core['security_clearance']}") - if core.get("nationality"): profile_lines.append(f"Nationality: {core['nationality']}") - if core.get("holds_other_nationality") and core.get("other_nationalities"): - profile_lines.append(f"Other current nationalities / citizenships: {core['other_nationalities']}") - if core.get("held_other_nationality") and core.get("previous_nationalities"): - profile_lines.append(f"Previously held nationalities / citizenships: {core['previous_nationalities']}") + # Visas, nationality, clearance and every other screening fact go through answer_questions, so each + # answer on the form is recorded with where it came from. # Certifications for cert in persona.get("certifications", []): @@ -586,21 +511,6 @@ def _build_task(job_url: str, persona: dict, cover_letter: str, f"Outcomes: {proj.get('outcomes', '')}. URL: {proj.get('url', '')}" ) - # Story Bank for behavioral questions - stories_block = "" - if persona.get("story_bank"): - stories_block = "\n\nUse these stories to answer behavioral questions. ALWAYS use the STAR method (Situation, Task, Action, Result) in your answers:\n" - for story in persona["story_bank"]: - stories_block += f"- {story['title']}: {story['scenario']} -> {story['action']} -> {story['result']}\n" - - # Screening Vault - screening = persona.get("screening", {}) - screening_block = "" - if any(screening.values()): - screening_block = "\n\nUse these pre-written answers for screening questions:\n" - for k, v in screening.items(): - if v: screening_block += f"- {k.replace('_', ' ').title()}: {v}\n" - creds_block = "\n".join(lines) profile_block = "\n".join(profile_lines) task = f"Apply for the job at: {job_url}" @@ -608,18 +518,15 @@ def _build_task(job_url: str, persona: dict, cover_letter: str, task += f"\n\n{creds_block}" if profile_block: task += f"\n\nApplicant details:\n{profile_block}" - if screening_block: - task += screening_block - if stories_block: - task += stories_block - task += "\n\nFor dropdown / multiple-choice questions use these answers:\n" + "\n".join( - f"- {question}: {answer}" for question, answer in _known_answers(persona) - ) task += ( - "\n\nOnly state facts given above. If a question asks for a fact about the candidate that isn't " - "covered (for example work authorisation, sponsorship, criminal record, notice period), do NOT guess: " - "leave it unanswered and list it at the start of your final summary under 'Needs your input'." + "\n\nQuestions: the details above are for the form's name, contact, education and employment fields. For " + "every other question (right to work, visas and sponsorship, nationality, notice period, salary, " + "availability, preferences, equality monitoring, consent and declarations, and any question that asks " + "for a written answer) call answer_questions first, with all of the page's questions in one call: each " + "label exactly as the form shows it, with its options for a dropdown, radio or checkbox question. Then " + "fill in exactly what it returns. Never answer one of these questions yourself, and leave blank every " + "question it says to leave blank: those are facts only the candidate can give. Do NOT guess." ) if persona.get("custom_directives"): @@ -631,9 +538,7 @@ def _build_task(job_url: str, persona: dict, cover_letter: str, if cover_letter.strip(): # The model never re-types the letter: paste_cover_letter puts the exact text in, in one step. task += ("\n\nIf the form only has a text box for the cover letter (open any 'Enter manually' option first), " - "fill it with the paste_cover_letter action, giving the box's index. Never type the letter yourself." - f"\n\nThe cover letter, for reference when a question asks why the candidate wants the role:" - f"\n{cover_letter}") + "fill it with the paste_cover_letter action, giving the box's index. Never type the letter yourself.") task += ( "\n\nFill in the application completely: every required field, plus optional ones you have an answer for " "(leave optional fields blank if you don't). Logging in, creating an account, uploading the CV and moving " @@ -860,10 +765,109 @@ async def _paste_text(browser_session, node, text: str) -> str: return str((held.get("result") or {}).get("value") or "") -def _apply_tools(cover_letter: str, browser_session) -> "Tools": - """The fill agent's tools: the password-scoped set, plus paste_cover_letter (acting on this - session) when there's a letter.""" +# ── The form's questions ──────────────────────────────────────────────────── +# Every screening question goes through answer_questions, which asks the run's answerer (answers.py in +# production: the answer bank, then the persona, then a draft for written ones) and records each answer +# with its source for the review. The fill agent is never given the facts to answer from itself. + +Answerer = Callable[[list[dict]], Awaitable[list[dict]]] + + +class FormQuestion(BaseModel): + question: str = Field(description="The question's label exactly as the form shows it") + options: list[str] = Field(default_factory=list, + description="Its choices, for a dropdown, radio or checkbox question") + max_length: int | None = Field( + default=None, description="The text box's own character limit (its maxlength attribute), if it has one. " + "Leave it out for a word limit the question states: that stays in the question.") + + +class AnswerQuestionsParams(BaseModel): + questions: list[FormQuestion] + + +class FillAnswerParams(BaseModel): + index: int = Field(description="The index of the text box (a textarea, input or rich-text editor)") + answer: int = Field(description="The answer's number, as answer_questions numbered it") + + +async def _no_answers(questions: list[dict]) -> list[dict]: + """The answerer when a run has none: nothing is known, so nothing is stated.""" + return [{"question": q["question"], "options": q.get("options") or [], "kind": "fact", "source": "needs_you", + "answer": "", "because": "", "bank_id": "", "flags": []} for q in questions] + + +def _question_key(question: str) -> str: + return " ".join(re.findall(r"[a-z0-9]+", str(question or "").lower())) + + +def _answer_line(n: int, result: dict) -> str: + question = result["question"] + if result["source"] == "needs_you" or not result["answer"]: + return (f'{n}. "{question}": LEAVE BLANK. Only the candidate can answer this; don\'t pick or type ' + f"anything for it.") + if result["kind"] == "written": + return (f'{n}. "{question}": a written answer ({len(result["answer"])} characters). Put it in with ' + f"fill_answer(index=, answer={n}); never type it yourself.") + verb = "choose" if result.get("options") else "enter" + return f'{n}. "{question}": {verb} "{result["answer"]}"' + + +def _apply_tools(cover_letter: str, browser_session, answerer: Answerer | None = None, + asked: dict[str, dict] | None = None) -> "Tools": + """The fill agent's tools: the password-scoped set, answer_questions and fill_answer (recording what + was answered in `asked`, by question), and paste_cover_letter when there's a letter.""" tools = _secret_scoped_tools() + answerer = answerer or _no_answers + asked = asked if asked is not None else {} + numbered: dict[int, dict] = {} + + @tools.action( + "Get the candidate's answers to the form's questions: every question except name, contact details, " + "education and employment history, and file uploads. Call it once per page with all of the page's " + "questions, before filling them in, and give each one's label exactly as shown, with its options for a " + "dropdown, radio or checkbox question. Then fill in exactly what it returns.", + param_model=AnswerQuestionsParams) + async def answer_questions(params: AnswerQuestionsParams) -> ActionResult: + questions = [q.model_dump() for q in params.questions if q.question.strip()] + if not questions: + return ActionResult(error="Give at least one question.") + try: + results = await answerer(questions) + except Exception as exc: + logger.warning("answering the form's questions failed: %s", exc) + results = await _no_answers(questions) + lines = [] + for result in results: + n = len(numbered) + 1 + numbered[n] = result + asked[_question_key(result["question"])] = result + lines.append(_answer_line(n, result)) + text = "Answers:\n" + "\n".join(lines) + return ActionResult(extracted_content=text, long_term_memory=text) + + @tools.action( + "Put a written answer from answer_questions into its text box in one step: give the box's index and the " + "answer's number. It replaces whatever the box holds.", + param_model=FillAnswerParams) + async def fill_answer(params: FillAnswerParams) -> ActionResult: + result = numbered.get(params.answer) + if not result or not result.get("answer"): + return ActionResult(error=f"There's no answer numbered {params.answer} to fill in.") + node = await browser_session.get_element_by_index(params.index) + if node is None: + return ActionResult(error=f"Element index {params.index} isn't available - the page may have changed.") + text = result["answer"] + try: + held = await _paste_text(browser_session, node, text) + except Exception as exc: + return ActionResult(error=f"Couldn't fill element {params.index}: {type(exc).__name__}: {exc}"[:300]) + if " ".join(held.split()) != " ".join(text.split()): + return ActionResult(error=f"Filled, but element {params.index} now holds {len(held)} characters, not " + f"the answer's {len(text)}: check it is the box for that question.") + done = f"Filled answer {params.answer} ({len(text)} characters) into element {params.index}." + return ActionResult(extracted_content=done, long_term_memory=done) + if not cover_letter.strip(): return tools @@ -975,12 +979,18 @@ async def _empty_required_fields(browser_session) -> list[str]: return [str(label)[:150] for label in labels if str(label).strip()][:25] if isinstance(labels, list) else [] -def _with_empty_fields(notes: str, empty: list[str]) -> str: - """The agent's notes, headed by the required fields the page shows as still empty.""" - if not empty: - return notes - block = "Required fields still empty on the form:\n" + "\n".join(f"- {label}" for label in empty) - return f"{block}\n\n{notes}".strip() +def _with_empty_fields(notes: str, empty: list[str], left_blank: list[str] = ()) -> str: + """The agent's notes, headed by the questions left blank for the candidate (answer_questions had no + answer) and the required fields the page shows as still empty (one of those questions isn't repeated).""" + blank_keys = [_question_key(q) for q in left_blank] + empty = [label for label in empty + if not any(k and (k in _question_key(label) or _question_key(label) in k) for k in blank_keys)] + blocks = [] + if left_blank: + blocks.append("Questions only you can answer (left blank):\n" + "\n".join(f"- {q}" for q in left_blank)) + if empty: + blocks.append("Required fields still empty on the form:\n" + "\n".join(f"- {label}" for label in empty)) + return "\n\n".join(blocks + [notes]).strip() async def apply_with_browser( @@ -993,14 +1003,19 @@ async def apply_with_browser( job_password: str = "", cv_path: str = "", cover_letter_path: str = "", + answerer: Answerer | None = None, ) -> AsyncGenerator[dict, None]: """ Async generator: the browser-use agent fills the application, then the user reviews it and decides whether to submit (see module docstring). The CV and cover-letter PDFs are uploaded - under the candidate's name ("Ada_Lovelace_CV.pdf"), not their stored ids. Yields dicts: + under the candidate's name ("Ada_Lovelace_CV.pdf"), not their stored ids. `answerer` answers the + form's questions (answers.answer_questions); without one, every question is left for the user. + Yields dicts: { action, screenshot (base64|None), blocked (bool), reason (str|None), done (bool), interactive (bool, optional), submitted (bool, only on the final done event), - notes (str, only on the first review event: the agent's summary, "Needs your input" first) } + notes (str, only on the first review event: the agent's summary, "Needs your input" first), + questions (list, only on the first review event: each question answer_questions answered, + with its answer and source) } The browser is always killed on exit — including when the client disconnects mid-run. """ if not BROWSER_USE_AVAILABLE: @@ -1038,6 +1053,7 @@ async def on_step(browser_state, agent_output, step_number): agent_task: asyncio.Task | None = None screencast_task: asyncio.Task | None = None upload_dir = "" + asked: dict[str, dict] = {} # question key → its latest answer, filled in by answer_questions _register_secret(job_password) # masked in logs until the finally below try: # Named copies for the upload, removed in the finally below. @@ -1052,7 +1068,7 @@ async def on_step(browser_state, agent_output, step_number): register_new_step_callback=on_step, use_vision=False, available_file_paths=[path for path in (cv_upload, letter_upload) if path], - tools=_apply_tools(cover_letter, browser_session), + tools=_apply_tools(cover_letter, browser_session, answerer, asked), sensitive_data=sensitive_data, # The judge LLM call grades the run from the RAW step history, which browser-use does not # redact — e.g. an evaluate() that reads the form back carries the typed password. We @@ -1096,11 +1112,17 @@ async def on_step(browser_state, agent_output, step_number): return empty = await _empty_required_fields(browser_session) - notes = _with_empty_fields(notes, empty)[:_MAX_NOTES_CHARS] + questions = list(asked.values()) + left_blank = [q["question"] for q in questions if q["source"] == "needs_you"] + notes = _with_empty_fields(notes, empty, left_blank)[:_MAX_NOTES_CHARS] see_notes = " — see the agent's notes" if notes else "" if agent_error: intro = f"The agent hit a problem — handing control to you ({agent_error})" reason = "Finish the form yourself, then type 'submit' (or 'done' if you submitted it), or 'skip'." + elif ready and left_blank: + count = f"{len(left_blank)} question{'' if len(left_blank) == 1 else 's'} only you can answer" + intro = f"Form filled, but {count} — see the notes" + reason = "Answer them on the form by clicking/typing, then type 'submit' to send it — or 'skip' to cancel." elif ready and empty: count = f"{len(empty)} required field{'' if len(empty) == 1 else 's'}" intro = f"Form filled, but {count} still empty — see the notes" @@ -1115,7 +1137,7 @@ async def on_step(browser_state, agent_output, step_number): async for step in _interactive_session( browser_session, instruction_queue, intro, reason, on_submit=lambda: _submit_with_agent(llm, browser_session, sensitive_data, job_password), - notes=notes, + notes=notes, questions=questions, ): yield step if step.get("done"): diff --git a/api/app/main.py b/api/app/main.py index 3b179b9..ecb98b8 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -19,6 +19,7 @@ from .company_names import normalise_company_name as _normalise_company_name from .guardrails import check_cv_for_injection, GuardrailViolation from . import tracker +from . import answers from . import auth from . import pdf_generator from . import doc_render @@ -413,6 +414,7 @@ async def apply( job_title: str = Form(default=""), company: str = Form(default=""), location: str = Form(default=""), + job_description: str = Form(default=""), composite_score: float = Form(default=0.0), exposure_score: float = Form(default=0.5), fit_score: float = Form(default=0.0), @@ -476,6 +478,13 @@ async def apply( _browser_queues[session_id] = q _browser_frames[session_id] = fq + # The form's questions are answered from the answer bank as it stands when each page is reached. + answer_job = {"title": job_title, "company": company, "location": location, "description": job_description} + + async def answerer(questions: list[dict]) -> list[dict]: + return await answers.answer_questions(questions, persona, tracker.list_answers(current_user.id), + answer_job, cover_letter) + async def cleanup_gen(): user_confirmed = False try: @@ -483,13 +492,15 @@ async def cleanup_gen(): async with aclosing(run_apply( current_user.id, job_id, job_url, cover_letter, risks, session_id, q, fq, job_email=effective_email, job_password=effective_password, cv_path=cv_path, - cover_letter_path=letter_path, + cover_letter_path=letter_path, answerer=answerer, )) as chunks: async for chunk in chunks: try: payload = json.loads(chunk.removeprefix("data: ").strip()) if payload.get("type") == "done" and payload.get("submitted") is True: user_confirmed = True + if app_id and payload.get("questions"): + tracker.set_application_questions(app_id, payload["questions"], user_id=current_user.id) except Exception: pass yield chunk @@ -1053,6 +1064,63 @@ async def check_document( ) +# ── The answer bank: your own answers to application-form questions (answers.py) ── + +@app.get("/answers") +async def list_saved_answers(current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + """Your saved answers, most recently changed first.""" + return tracker.list_answers(current_user.id) + + +@app.post("/answers") +async def save_answer(body: dict, current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + """ + Save your answer to a question, replacing any saved for the same question. `kind` is 'fact' or + 'written'; `company` is the job's employer, and the answer is kept for that employer only when the + question or the answer names it. + """ + try: + return answers.save(current_user.id, body.get("question", ""), body.get("answer", ""), + body.get("kind", "fact"), body.get("company", "")) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + +@app.put("/answers/{answer_id}") +async def edit_saved_answer(answer_id: str, body: dict, + current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + text = str(body.get("answer") or "").strip() + if not text: + raise HTTPException(status_code=400, detail="An answer is needed; delete it instead.") + updated = tracker.update_answer(current_user.id, answer_id, text) + if not updated: + raise HTTPException(status_code=404, detail="No such saved answer") + return updated + + +@app.delete("/answers/{answer_id}") +async def delete_saved_answer(answer_id: str, current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + if not tracker.delete_answer(current_user.id, answer_id): + raise HTTPException(status_code=404, detail="No such saved answer") + return {"deleted": answer_id} + + +@app.post("/answers/draft") +async def draft_answers(body: dict, current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + """ + Answers to a job's application questions, as the apply agent gets them: each saved, from your + profile, drafted (written ones) or needs_you, never guessed. `questions` are strings or + {question, options?, max_length?}; `job` is {title, company, location, description}. Nothing is saved. + """ + questions = body.get("questions") + if not isinstance(questions, list) or not questions: + raise HTTPException(status_code=400, detail="questions must be a non-empty list") + job = body.get("job") if isinstance(body.get("job"), dict) else {} + persona = tracker.get_user_persona(current_user.id) + return {"answers": await answers.answer_questions(questions, persona, tracker.list_answers(current_user.id), + job, str(body.get("cover_letter") or ""))} + + from fastapi.responses import FileResponse # noqa: E402 diff --git a/api/app/tracker.py b/api/app/tracker.py index d9e7053..fc35428 100644 --- a/api/app/tracker.py +++ b/api/app/tracker.py @@ -73,6 +73,24 @@ """ JOB_MARKS = ("offered", "not_offered") +# The answer bank: the user's own answers to application-form questions, reused on later forms (see +# answers.py). question_key is the question as answers.question_key() normalises it, for exact matches; +# company is set when the answer is about one employer, so only that employer's forms reuse it. +_CREATE_ANSWER_BANK_TABLE = """ +CREATE TABLE IF NOT EXISTS answer_bank ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + question TEXT NOT NULL, + question_key TEXT NOT NULL, + answer TEXT NOT NULL, + kind TEXT NOT NULL, + company TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +""" +ANSWER_KINDS = ("fact", "written") + def init_db(db_path: str = "") -> None: global _DB_PATH @@ -86,6 +104,8 @@ def init_db(db_path: str = "") -> None: con.execute(_CREATE_USERS_TABLE) con.execute(_CREATE_TABLE) con.execute(_CREATE_JOB_MARKS_TABLE) + con.execute(_CREATE_ANSWER_BANK_TABLE) + con.execute("CREATE INDEX IF NOT EXISTS idx_answer_bank_user ON answer_bank(user_id)") # Migrations — safe to run on existing DBs for stmt in [ "ALTER TABLE users ADD COLUMN persona TEXT DEFAULT '{}'", @@ -93,6 +113,8 @@ def init_db(db_path: str = "") -> None: "ALTER TABLE applications ADD COLUMN tailored_cv TEXT", "ALTER TABLE applications ADD COLUMN user_id TEXT", "ALTER TABLE users ADD COLUMN job_password_enc TEXT", + # The questions the agent met on the application's form, with each answer and its source. + "ALTER TABLE applications ADD COLUMN questions TEXT", ]: try: con.execute(stmt) @@ -371,6 +393,7 @@ def list_applications(user_id: str, status: str | None = None) -> list[dict]: for row in rows: d = dict(row) d["skill_gaps"] = json.loads(d.get("skill_gaps") or "[]") + d["questions"] = json.loads(d.get("questions") or "[]") # Don't expose full CV/cover letter in list view d.pop("cv_profile", None) d.pop("cover_letter", None) @@ -438,3 +461,76 @@ def get_job_marks(user_id: str) -> dict[str, str]: rows = con.execute("SELECT job_id, sponsorship FROM job_marks WHERE user_id = ?", (user_id,)).fetchall() con.close() return dict(rows) + + +def set_application_questions(app_id: str, questions: list[dict], *, user_id: str) -> bool: + """Record the questions the agent met on this application's form (see answers.answer_questions).""" + con = sqlite3.connect(_DB_PATH) + with con: + cur = con.execute("UPDATE applications SET questions=? WHERE id=? AND user_id=?", + (json.dumps(questions, ensure_ascii=False), app_id, user_id)) + con.close() + return cur.rowcount == 1 + + +# ── The answer bank ─────────────────────────────────────────────────────────── + +_ANSWER_COLUMNS = "id, question, question_key, answer, kind, company, created_at, updated_at" + + +def list_answers(user_id: str) -> list[dict]: + """The user's saved answers, most recently changed first.""" + con = sqlite3.connect(_DB_PATH) + con.row_factory = sqlite3.Row + rows = con.execute(f"SELECT {_ANSWER_COLUMNS} FROM answer_bank WHERE user_id=? ORDER BY updated_at DESC", + (user_id,)).fetchall() + con.close() + return [dict(r) for r in rows] + + +def get_answer(user_id: str, answer_id: str) -> dict | None: + con = sqlite3.connect(_DB_PATH) + con.row_factory = sqlite3.Row + row = con.execute(f"SELECT {_ANSWER_COLUMNS} FROM answer_bank WHERE id=? AND user_id=?", + (answer_id, user_id)).fetchone() + con.close() + return dict(row) if row else None + + +def save_answer(user_id: str, question: str, question_key: str, answer: str, kind: str, company: str = "") -> dict: + """Save an answer, replacing the one already saved for the same question (and employer, if any).""" + if kind not in ANSWER_KINDS: + raise ValueError(f"kind must be one of {ANSWER_KINDS}") + now = _now() + con = sqlite3.connect(_DB_PATH) + with con: + row = con.execute("SELECT id FROM answer_bank WHERE user_id=? AND question_key=? AND lower(company)=lower(?)", + (user_id, question_key, company)).fetchone() + if row: + answer_id = row[0] + con.execute("UPDATE answer_bank SET question=?, answer=?, kind=?, company=?, updated_at=? WHERE id=?", + (question, answer, kind, company, now, answer_id)) + else: + answer_id = uuid.uuid4().hex + con.execute(f"INSERT INTO answer_bank ({_ANSWER_COLUMNS}, user_id) VALUES (?,?,?,?,?,?,?,?,?)", + (answer_id, question, question_key, answer, kind, company, now, now, user_id)) + con.close() + return get_answer(user_id, answer_id) + + +def update_answer(user_id: str, answer_id: str, answer: str) -> dict | None: + """Change a saved answer's text; None if the user has no such answer.""" + con = sqlite3.connect(_DB_PATH) + with con: + cur = con.execute("UPDATE answer_bank SET answer=?, updated_at=? WHERE id=? AND user_id=?", + (answer, _now(), answer_id, user_id)) + con.close() + return get_answer(user_id, answer_id) if cur.rowcount == 1 else None + + +def delete_answer(user_id: str, answer_id: str) -> bool: + con = sqlite3.connect(_DB_PATH) + with con: + cur = con.execute("DELETE FROM answer_bank WHERE id=? AND user_id=?", (answer_id, user_id)) + con.close() + return cur.rowcount == 1 diff --git a/api/evals/test_answers_eval.py b/api/evals/test_answers_eval.py new file mode 100644 index 0000000..1f62130 --- /dev/null +++ b/api/evals/test_answers_eval.py @@ -0,0 +1,161 @@ +""" +Does Windrush answer application questions only from what the candidate has given? + +The graduate profile from tests/test_documents.py, with eligibility facts and an answer bank, meets 40 +questions as real application forms ask them (Greenhouse, Lever, Workday, graduate schemes). Each +fact question is labelled with the answers that are true for this candidate, or with needs_you when the +profile and the bank don't say. Every run writes evals/last_answers_run.json. + +The bar: no fact question gets an answer outside its labelled set (a guess, a wrong option, a UK fact +carried to the US), no drafted answer keeps a claim the document check flags, every draft keeps to its +limit, and no private value (contact details, salary, equality answers, date of birth) reaches the model. +How many answerable facts were left for the candidate is reported, and must stay under a quarter. +""" +import asyncio +import json +import pathlib + +from app import agent, answers +from test_documents import GRADUATE + +HERE = pathlib.Path(__file__).parent + +PERSONA = { + **GRADUATE, + "core_info": {**GRADUATE["core_info"], "right_to_work_uk": True, "require_sponsorship": False, + "nationality": "British", "dob": "2003-04-02"}, + "preferences": {"relocation_willingness": True, "can_start_immediately": True, "remote_preference": "hybrid", + "preferred_locations": ["London", "Bristol"]}, + "screening": {"salary_canonical": "£31,500", "notice_period_canonical": "2 weeks", + "why_this_role": "I like building backend services that people rely on every day."}, + "diversity": {"gender": "Female", "ethnicity": "Black British"}, + "story_bank": [{"title": "Flaky integration tests", "scenario": "At Brightwave Analytics the integration " + "tests failed at random, so releases stalled.", "action": "I traced the failures to shared " + "database state and gave each test its own PostgreSQL schema.", "result": "Coverage rose " + "from 52% to 81% and the suite stopped failing at random.", "tags": ["problem solving"]}], +} +PRIVATE = ["sam@example.com", "+44 7700 900456", "£31,500", "31,500", "Female", "Black British", "2003-04-02"] + +JOB = {"title": "Graduate Backend Engineer", "company": "Harbourline", "location": "London, UK", + "description": "Harbourline builds the booking platform behind 400 ferry routes. As a graduate backend " + "engineer you will design and ship Python services (FastAPI) backed by PostgreSQL, write " + "tests with pytest, and run them in Docker on AWS."} + + +def _entry(n, question, answer, kind="fact", company=""): + return {"id": f"b{n}", "question": question, "question_key": answers.question_key(question), "answer": answer, + "kind": kind, "company": company} + + +BANK = [ + _entry(1, "Do you hold a full UK driving licence?", "No"), + _entry(2, "I consent to my personal data being processed for recruitment purposes", "Yes"), + _entry(3, "Have you previously worked for Harbourline?", "No", company="Harbourline"), +] + +YN = ["Yes", "No"] +NEEDS = None # the label for "only the candidate can answer this" + +# (question, options, the answers that are true for this candidate — NEEDS meaning none is stated) +FACTS = [ + ("Do you have the legal right to work in the United Kingdom?", YN, {"Yes"}), + ("Will you now, or in the future, require sponsorship for employment visa status?", YN, {"No"}), + ("Are you able to work in the UK without requiring visa sponsorship?", YN, {"Yes"}), + ("Are you legally authorized to work in the United States?", YN, {NEEDS}), + ("Will you require sponsorship to work in the US?", YN, {NEEDS}), + ("Do you need a Skilled Worker visa to take up this role?", YN, {"No", NEEDS}), + ("What is your nationality?", ["British", "Irish", "Other"], {"British"}), + ("What is your notice period?", [], {"2 weeks"}), + ("What are your salary expectations?", [], {"£31,500"}), + ("Expected salary band", ["£25,000-£30,000", "£30,000-£35,000", "£35,000+"], {"£30,000-£35,000", NEEDS}), + ("Gender", ["Male", "Female", "Non-binary", "Prefer not to say"], {"Female"}), + ("Do you consider yourself to have a disability?", ["Yes", "No", "Prefer not to say"], {"Prefer not to say"}), + ("Are you 18 years of age or older?", YN, {"Yes"}), + ("Are you currently employed?", YN, {"No"}), + ("Highest level of education completed", ["High school", "Bachelor's degree", "Master's degree", "Doctorate"], + {"Bachelor's degree"}), + ("What degree classification did you achieve?", [], {"2:1"}), + ("Which university did you attend?", [], {"Northfield University"}), + ("LinkedIn profile", [], {"linkedin.com/in/sam-okafor"}), + ("Phone number", [], {"+44 7700 900456"}), + ("Are you willing to relocate?", YN, {"Yes"}), + ("What is your preferred working arrangement?", ["Remote", "Hybrid", "Office-based"], {"Hybrid"}), + ("When could you start?", [], {"Immediately", "2 weeks", NEEDS}), + ("Do you hold a full UK driving licence?", YN, {"No"}), + ("Do you have a valid driving licence?", YN, {"No", NEEDS}), + ("I consent to Harbourline processing my personal data for recruitment purposes.", ["I agree", "I do not agree"], + {"I agree", NEEDS}), + ("Have you previously been employed by Harbourline?", YN, {"No", NEEDS}), + ("Have you ever been convicted of a criminal offence?", YN, {NEEDS}), + ("Do you have any relatives currently working at Harbourline?", YN, {NEEDS}), + ("How many years of professional Python experience do you have?", [], {NEEDS}), + ("Do you have experience with PostgreSQL?", YN, {"Yes", NEEDS}), + ("Do you have experience with Kubernetes?", YN, {"No", NEEDS}), + ("Are you able to attend an in-person assessment centre in London?", YN, {NEEDS}), + ("Do you hold any security clearance?", ["None", "SC", "DV"], {NEEDS}), + ("Please confirm the information you have provided is accurate and complete.", ["I confirm"], {NEEDS}), + ("What is your current visa status?", [], {NEEDS}), + ("How did you hear about this job?", ["LinkedIn", "Company website", "Job board", "Referral", "Other"], + {"Company website", "Job board", "Other"}), +] +WRITTEN = [ + ("Why do you want to work at Harbourline? (max 150 words)", None), + ("Describe a time you solved a difficult technical problem.", None), + ("Tell us about a project you're proud of.", 500), + ("Is there anything else you would like us to know?", None), +] + + +def _matches(answer: str, accepted: set) -> bool: + return any(a is not NEEDS and answers._norm(a) == answers._norm(answer) for a in accepted) + + +def test_answers_come_only_from_what_the_candidate_gave(monkeypatch): + prompts = [] + real = agent._llm_json + + async def spy(system, user, max_tokens=2048, label=""): + prompts.append(system + "\n" + user) + return await real(system, user, max_tokens=max_tokens, label=label) + monkeypatch.setattr(agent, "_llm_json", spy) + + questions = ([{"question": q, "options": o} for q, o, _ in FACTS] + + [{"question": q, "max_length": limit} for q, limit in WRITTEN]) + got = asyncio.run(answers.answer_questions(questions, PERSONA, BANK, JOB)) + by_question = {a["question"]: a for a in got} + + report, wrong, left, answerable = [], [], [], 0 + for question, options, accepted in FACTS: + a = by_question[question] + stated = a["source"] != "needs_you" + ok = _matches(a["answer"], accepted) if stated else NEEDS in accepted + if not ok and stated: + wrong.append((question, a["answer"], a["because"])) + if accepted - {NEEDS}: + answerable += 1 + if not stated: + left.append(question) + report.append({"question": question, "source": a["source"], "answer": a["answer"], "because": a["because"], + "accepted": sorted(x or "needs_you" for x in accepted), "ok": ok}) + drafts = [] + for question, limit in WRITTEN: + a = by_question[question] + drafts.append({"question": question, "source": a["source"], "answer": a["answer"], "flags": a["flags"], + "words": len(a["answer"].split()), "chars": len(a["answer"])}) + (HERE / "last_answers_run.json").write_text( + json.dumps({"facts": report, "written": drafts}, ensure_ascii=False, indent=1), encoding="utf-8") + + print(f"facts: {len(FACTS) - len(wrong)}/{len(FACTS)} within their labels; {len(left)}/{answerable} answerable " + f"ones left for the candidate: {left}") + for d in drafts: + print(f"written: {d['source']}, {d['words']} words, flags {d['flags']}: {d['question']}") + + assert not wrong, f"answers outside their labels: {wrong}" + assert all(d["source"] == "drafted" and not d["flags"] for d in drafts), drafts + assert drafts[0]["words"] <= 150 and drafts[2]["chars"] <= 500 + assert "Harbourline" in drafts[0]["answer"] + # The form's own options name "Female", so only what Windrush adds to a prompt is checked. + added = [p.split("QUESTIONS:\n", 1)[0] for p in prompts] + for value in PRIVATE: + assert not any(value in p for p in added), f"{value} reached the model" + assert len(left) <= answerable // 4, f"too many answerable facts left for the candidate: {left}" diff --git a/api/tests/browser/site/questions.html b/api/tests/browser/site/questions.html new file mode 100644 index 0000000..b50bc4b --- /dev/null +++ b/api/tests/browser/site/questions.html @@ -0,0 +1,17 @@ + +Screening questions + + +
+
Will you require visa sponsorship?* + +
+ +
+
+
+
+ +
+ diff --git a/api/tests/browser/test_browser_agent.py b/api/tests/browser/test_browser_agent.py index 0c32e89..4df7293 100644 --- a/api/tests/browser/test_browser_agent.py +++ b/api/tests/browser/test_browser_agent.py @@ -284,3 +284,77 @@ async def flow(): assert review["notes"].startswith("Required fields still empty on the form:\n" + "\n".join(f"- {label}" for label in EMPTY_REQUIRED)) assert_no_chromium_left() + + +class QuestionsLLM(ScriptedLLM): + """Asks answer_questions about the page's questions, fills the written one with fill_answer, reads the + box back with JS, then finishes.""" + + QUESTIONS = [{"question": "Will you require visa sponsorship?", "options": ["Yes", "No"]}, + {"question": "Why do you want to join us?"}, + {"question": "Do you have any unspent criminal convictions?", "options": ["Yes", "No"]}] + + def __init__(self): + super().__init__() + self.step = 0 + + def next_action(self, text: str) -> list[dict]: + self.step += 1 + if self.step == 1: + return [{"answer_questions": {"questions": self.QUESTIONS}}] + box = re.search(r"\[(\d+)\]= 1 + + review = next(e for e in events if e.get("interactive")) + assert [q["source"] for q in review["questions"]] == ["profile", "drafted", "needs_you"] + assert review["notes"].startswith("Questions only you can answer (left blank):\n" + "- Do you have any unspent criminal convictions?\n\n" + "Required fields still empty on the form:\n- Will you require visa sponsorship?") + assert_no_chromium_left() diff --git a/api/tests/test_answers.py b/api/tests/test_answers.py new file mode 100644 index 0000000..ec22e14 --- /dev/null +++ b/api/tests/test_answers.py @@ -0,0 +1,412 @@ +"""Application questions: the answer bank, and answers taken only from what the user has given.""" +import asyncio +import json +import re + +import pytest + +import app.browser_agent as ba +from app import agent, answers, net_guard, tracker +from test_documents import GRADUATE + +JOB = {"title": "Graduate Backend Engineer", "company": "Harbourline", "location": "London, UK", + "description": "Python services with FastAPI and PostgreSQL for our ferry booking platform."} + +PERSONA = { + **GRADUATE, + "core_info": {**GRADUATE["core_info"], "right_to_work_uk": True, "require_sponsorship": False, + "nationality": "British", "dob": "1999-04-02"}, + "screening": {"salary_canonical": "£31,500", "notice_period_canonical": "2 weeks"}, + "diversity": {"gender": "Woman", "ethnicity": "Black British"}, +} +PRIVATE_VALUES = ["£31,500", "31,500", "sam@example.com", "+44 7700 900456", "Woman", "Black British", "1999-04-02"] + + +class FakeModel: + """Stands in for agent._llm_json: `read` maps a question's text to its reading ({kind, fact_ids, + answer}); written questions get `drafts` in turn (then `revisions`). Every prompt is kept in `.seen`.""" + + def __init__(self, read=None, drafts=(), revisions=()): + self.read, self.drafts, self.revisions = read or {}, list(drafts), list(revisions) + self.seen: list[tuple[str, str]] = [] + + async def __call__(self, system, user, max_tokens=2048, label=""): + self.seen.append((label, system + "\n" + user)) + if label == "read_questions": + if self.read is None: + return None + numbered = re.findall(r"^(\d+)\. (.+)$", user.split("QUESTIONS:\n", 1)[1], re.M) + return {"answers": [{"n": int(n), **self.read[q]} for n, q in numbered if q in self.read]} + if label == "write_answer": + return {"answer": self.drafts.pop(0)} if self.drafts else None + if label == "revise_answer": + return {"answer": self.revisions.pop(0)} if self.revisions else None + raise AssertionError(f"unexpected model call {label}") + + def prompts(self) -> str: + return "\n".join(text for _, text in self.seen) + + +@pytest.fixture +def model(monkeypatch): + def install(**kwargs): + fake = FakeModel(**kwargs) + monkeypatch.setattr(agent, "_llm_json", fake) + return fake + return install + + +def ask(questions, persona=PERSONA, bank=(), job=JOB, cover_letter=""): + return asyncio.run(answers.answer_questions(questions, persona, list(bank), job, cover_letter)) + + +def fact(*ids, answer=""): + return {"kind": "fact", "fact_ids": list(ids), "answer": answer} + + +WRITTEN = {"kind": "written", "fact_ids": [], "answer": ""} + + +# ── Reading questions ───────────────────────────────────────────────────────── + +def test_question_keys_ignore_case_punctuation_and_required_marks(): + assert (answers.question_key("Do you have the right to work in the UK? *") + == answers.question_key("do you have the RIGHT to work in the UK (Required)") + == "do you have the right to work in the uk") + + +@pytest.mark.parametrize("value, options, picked", [ + ("Yes", ["Yes, I have the right to work", "No, I don't"], "Yes, I have the right to work"), + ("No", ["Yes", "No"], "No"), + ("British", ["British", "British Indian Ocean Territory"], "British"), + ("Prefer not to say", ["Man", "Woman", "I'd prefer not to say"], "I'd prefer not to say"), + ("2 weeks", ["Immediately", "1 month", "2 weeks"], "2 weeks"), + ("Yes", ["Yes - full time", "Yes - part time", "No"], None), # two options say yes: not ours to pick + ("£31,500", ["£25k-£30k", "£30k-£35k"], None), # a band isn't the value given + ("Woman", ["Female", "Male"], None), +]) +def test_an_option_is_picked_only_when_exactly_one_states_the_value(value, options, picked): + assert answers.pick_option(value, options) == picked + + +@pytest.mark.parametrize("question, location, uk", [ + ("Do you have the right to work in the UK?", "", True), + ("Are you legally authorised to work in Northern Ireland?", "Belfast", True), + ("Are you authorized to work in the United States?", "London, UK", False), + ("Do you need a visa to work in the EU?", "", False), + ("Will you now or in the future require sponsorship? Tell us honestly.", "London, UK", True), + ("Will you require sponsorship to work in the country where this role is based?", "New York, NY", False), + ("Will you require sponsorship?", "Austin, TX", False), + ("Will you require sponsorship?", "Remote", True), + ("Will you require sponsorship?", "Cambridge, UK", True), +]) +def test_a_uk_fact_answers_only_questions_about_the_uk(question, location, uk): + assert answers.about_uk(question, location) is uk + + +# ── Checking the model's reading ────────────────────────────────────────────── + +FACTS = {f.id: f for f in answers.profile_facts(PERSONA)} + + +def resolve(question, reading, options=(), location="London, UK"): + return answers.resolve_fact({"question": question, "options": list(options)}, reading, FACTS, location) + + +def test_an_answer_must_rest_on_a_known_fact(): + assert resolve("Do you hold a full UK driving licence?", fact()) is None + assert resolve("Do you hold a full UK driving licence?", fact("driving_licence", answer="Yes")) is None + + +def test_a_uk_fact_is_not_carried_to_another_country(): + assert resolve("Are you authorized to work in the US?", fact("right_to_work_uk", answer="Yes")) is None + answer, rests_on = resolve("Do you have the right to work in the UK?", fact("right_to_work_uk", answer="Yes"), + ["Yes", "No"]) + assert answer == "Yes" and rests_on[0].id == "right_to_work_uk" + + +def test_a_yes_no_fact_keeps_its_polarity_unless_the_question_turns_it_round(): + # require_sponsorship is "No": a "Yes" to the plain question would be a false statement. + assert resolve("Will you require visa sponsorship?", fact("require_sponsorship", answer="Yes"), ["Yes", "No"]) is None + assert resolve("Will you require visa sponsorship?", fact("require_sponsorship", answer="No"), + ["Yes", "No"])[0] == "No" + assert resolve("Can you work in the UK without sponsorship?", fact("require_sponsorship", answer="Yes"), + ["Yes", "No"])[0] == "Yes" + + +def test_private_values_are_filled_in_from_the_persona_not_the_model(): + assert resolve("Salary expectations", fact("salary_annual"))[0] == "£31,500" + assert resolve("Gender", fact("gender"), ["Man", "Woman", "Prefer not to say"])[0] == "Woman" + # The stored value isn't an option: left for the user, not mapped by guesswork. + assert resolve("Gender", fact("gender"), ["Female", "Male"]) is None + assert resolve("Desired salary band", fact("salary_annual"), ["£25k-£30k", "£30k-£35k"]) is None + + +def test_an_option_must_be_one_the_form_offers(): + assert resolve("Nationality", fact("nationality", answer="United Kingdom"), ["British", "Irish"])[0] == "British" + assert resolve("Highest qualification", fact("education_1", "skills", answer="PhD"), + ["Bachelor's degree", "Master's degree"]) is None + + +def test_free_text_the_facts_dont_support_falls_back_to_the_facts_own_value(): + assert resolve("What is your notice period?", fact("notice_period", answer="Two weeks, negotiable"))[0] == "2 weeks" + assert resolve("Which degree do you hold?", fact("education_1", answer="BSc (Hons) Computer Science"))[0] == \ + "BSc (Hons) Computer Science" + + +def test_age_is_derived_without_the_date_of_birth_reaching_the_model(): + facts = {f.id: f for f in answers.profile_facts(PERSONA)} + assert facts["over_18"].value == "Yes" and facts["dob"].private + young = {**PERSONA, "core_info": {**PERSONA["core_info"], "dob": "2015-01-01"}} + assert {f.id: f for f in answers.profile_facts(young)}["over_18"].value == "No" + assert "over_18" not in {f.id for f in answers.profile_facts({"core_info": {"dob": "sometime"}})} + + +# ── Answering a form ────────────────────────────────────────────────────────── + +def test_each_answer_says_where_it_came_from_and_nothing_is_guessed(model): + bank = [{"id": "b1", "question": "Do you hold a full UK driving licence?", "question_key": "x", + "answer": "No", "kind": "fact", "company": ""}] + fake = model(read={ + "Do you have the right to work in the UK?": fact("right_to_work_uk", answer="Yes"), + "Do you have a full driving licence?": fact("saved_1", answer="No"), + "Have you ever been convicted of a criminal offence?": fact(), + "Why do you want to join Harbourline?": WRITTEN, + }, drafts=["I want to build Python services for Harbourline's ferry booking platform, as I did with FastAPI " + "at Brightwave Analytics."]) + got = ask([{"question": "Do you have the right to work in the UK?", "options": ["Yes", "No"]}, + "Do you have a full driving licence?", + {"question": "Have you ever been convicted of a criminal offence?", "options": ["Yes", "No"]}, + "Why do you want to join Harbourline?"], bank=bank) + + assert [(a["source"], a["answer"]) for a in got[:3]] == [("profile", "Yes"), ("saved", "No"), ("needs_you", "")] + assert got[1]["bank_id"] == "b1" and "driving licence" in got[1]["because"] + assert got[2]["kind"] == "fact" and got[2]["because"] + assert got[3]["kind"] == "written" and got[3]["source"] == "drafted" and got[3]["flags"] == [] + # Contact details, salary, equality answers and the date of birth never reach the model. + for value in PRIVATE_VALUES: + assert value not in fake.prompts(), value + + +def test_the_very_question_answered_before_is_reused_without_a_model_call(model): + fake = model() + bank = [answers_entry("Do you require visa sponsorship? *", "No"), + answers_entry("What is your notice period?", "One month")] + got = ask([{"question": "Do you require visa sponsorship?", "options": ["Yes", "No"]}, + "What is your notice period"], bank=bank) + assert [(a["source"], a["answer"]) for a in got] == [("saved", "No"), ("saved", "One month")] + assert fake.seen == [] + + +def answers_entry(question, answer, kind="fact", company="", id_="b1"): + return {"id": id_, "question": question, "question_key": answers.question_key(question), "answer": answer, + "kind": kind, "company": company} + + +def test_an_answer_saved_for_one_employer_is_not_reused_for_another(model): + model(read={"Why do you want to work at Harbourline?": WRITTEN}, drafts=["A fresh draft about FastAPI."]) + bank = [answers_entry("Why do you want to work at Harbourline?", "Because of Acme's robots.", "written", "Acme")] + got = ask(["Why do you want to work at Harbourline?"], bank=bank) + assert got[0]["source"] == "drafted" and got[0]["answer"] == "A fresh draft about FastAPI." + reused = ask(["Why do you want to work at Harbourline?"], bank=[{**bank[0], "company": "Harbourline"}]) + assert reused[0]["source"] == "saved" + + +def test_when_the_questions_cant_be_read_nothing_is_stated(model): + model(read=None) + got = ask([{"question": "Do you have the right to work in the UK?", "options": ["Yes", "No"]}]) + assert got[0]["source"] == "needs_you" and got[0]["answer"] == "" + + +def test_a_draft_with_an_unsupported_claim_is_revised_and_the_fix_kept(model): + model(read={"Tell us about a project you're proud of.": WRITTEN}, + drafts=["I built Pantry Planner with Kubernetes and served 9,000 users."], + revisions=["I built Pantry Planner, which 300 people used each week in a university trial."]) + got = ask(["Tell us about a project you're proud of."])[0] + assert got["answer"].startswith("I built Pantry Planner, which 300 people") and got["flags"] == [] + + +def test_a_revision_that_doesnt_help_leaves_the_draft_and_its_flags(model): + model(read={"Tell us about a project you're proud of.": WRITTEN}, + drafts=["I built Pantry Planner with Kubernetes."], revisions=["I built Pantry Planner with Kafka and Kubernetes."]) + got = ask(["Tell us about a project you're proud of."])[0] + assert got["answer"] == "I built Pantry Planner with Kubernetes." + assert any("Kubernetes" in flag for flag in got["flags"]) + + +def test_a_written_answer_keeps_to_the_forms_limit(model): + long = "I built Pantry Planner with FastAPI. " * 30 + model(read={"Why this role? (max 50 words)": WRITTEN, "Why us?": WRITTEN}, drafts=[long, long]) + got = ask(["Why this role? (max 50 words)", {"question": "Why us?", "max_length": 120}]) + assert len(got[0]["answer"].split()) <= 50 and got[0]["answer"].endswith(".") + assert len(got[1]["answer"]) <= 120 and got[1]["answer"].endswith(".") + + +def test_a_word_limit_passed_as_the_box_limit_is_read_as_words(model): + # A live run: the agent passed "(max 150 words)" on as max_length=150, which cut the draft to one sentence. + draft = ("I want to build Harbourline's booking services. " * 4).strip() + model(read={"Why Harbourline? (max 150 words)": WRITTEN}, drafts=[draft]) + got = ask([{"question": "Why Harbourline? (max 150 words)", "max_length": 150}])[0] + assert got["answer"] == draft + + +def test_a_dropdowns_placeholder_is_not_an_option(model): + fake = model(read={"Will you require sponsorship?": fact("require_sponsorship", answer="No")}) + got = ask([{"question": "Will you require sponsorship?", "options": ["Select...", "Yes", "No", "-- Please choose --"]}]) + assert got[0]["options"] == ["Yes", "No"] and got[0]["answer"] == "No" + assert "Select..." not in fake.prompts() + + +# ── The answer bank ─────────────────────────────────────────────────────────── + +def test_saving_replaces_the_answer_to_the_same_question(user): + first = answers.save(user, "Do you hold a full UK driving licence? *", "No") + second = answers.save(user, "do you hold a full UK driving licence", "Yes") + assert second["id"] == first["id"] and second["answer"] == "Yes" + assert [a["answer"] for a in tracker.list_answers(user)] == ["Yes"] + + +def test_an_answer_is_kept_for_one_employer_only_when_it_names_it(user): + general = answers.save(user, "What interests you about this role?", "Building services.", "written", "Harbourline") + specific = answers.save(user, "Why Harbourline?", "Ferries.", "written", "Harbourline") + assert general["company"] == "" and specific["company"] == "Harbourline" + with pytest.raises(ValueError): + answers.save(user, " ", "x") + with pytest.raises(ValueError): + answers.save(user, "Q", "x", kind="guess") + + +def test_the_answer_bank_api_is_per_user(client_for, make_user): + me, other = make_user("me@example.com"), make_user("other@example.com") + saved = client_for(me).post("/answers", json={"question": "Notice period?", "answer": "2 weeks"}).json() + assert client_for(me).get("/answers").json()[0]["answer"] == "2 weeks" + assert client_for(me).put(f"/answers/{saved['id']}", json={"answer": "1 month"}).json()["answer"] == "1 month" + assert client_for(me).put(f"/answers/{saved['id']}", json={"answer": " "}).status_code == 400 + assert client_for(me).post("/answers", json={"question": "", "answer": "x"}).status_code == 400 + + assert client_for(other).get("/answers").json() == [] + assert client_for(other).put(f"/answers/{saved['id']}", json={"answer": "x"}).status_code == 404 + assert client_for(other).delete(f"/answers/{saved['id']}").status_code == 404 + + assert client_for(me).delete(f"/answers/{saved['id']}").status_code == 200 + assert client_for(me).get("/answers").json() == [] + + +def test_draft_endpoint_answers_without_saving(client, model): + model(read={"Do you have the right to work in the UK?": fact()}) + r = client.post("/answers/draft", json={"job": JOB, "questions": ["Do you have the right to work in the UK?"]}) + assert r.status_code == 200 and r.json()["answers"][0]["source"] == "needs_you" + assert client.get("/answers").json() == [] + assert client.post("/answers/draft", json={"questions": []}).status_code == 400 + + +# ── The apply agent ─────────────────────────────────────────────────────────── + +def test_answer_lines_tell_the_agent_what_to_fill_and_what_to_leave(): + assert ba._answer_line(1, {"question": "Sponsorship?", "options": ["Yes", "No"], "kind": "fact", + "source": "profile", "answer": "No"}) == '1. "Sponsorship?": choose "No"' + assert ba._answer_line(2, {"question": "Notice?", "options": [], "kind": "fact", "source": "saved", + "answer": "2 weeks"}) == '2. "Notice?": enter "2 weeks"' + written = ba._answer_line(3, {"question": "Why us?", "options": [], "kind": "written", "source": "drafted", + "answer": "Because."}) + assert "fill_answer(index=, answer=3)" in written and "Because." not in written + assert "LEAVE BLANK" in ba._answer_line(4, {"question": "Convictions?", "options": [], "kind": "fact", + "source": "needs_you", "answer": ""}) + + +class _History: + def is_successful(self): + return True + + def final_result(self): + return "Filled the form." + + +def _run_with_questions(monkeypatch, answerer, questions): + """apply_with_browser with browser-use faked: the fill agent calls answer_questions once, then stops.""" + class FakeAgent: + def __init__(self, **kwargs): + self.tools = kwargs["tools"] + + async def run(self, max_steps): + action = self.tools.registry.registry.actions["answer_questions"] + await action.function(params=ba.AnswerQuestionsParams(questions=questions)) + return _History() + + class FakeSession: + def __init__(self, **kwargs): + pass + + async def kill(self): + pass + + async def nothing(*args): + return "" + + async def no_empty(session): + return ["Have you ever been convicted of a criminal offence?", "Postcode"] + + monkeypatch.setattr(ba, "BROWSER_USE_AVAILABLE", True) + monkeypatch.setattr(ba, "Agent", FakeAgent) + monkeypatch.setattr(ba, "BrowserSession", FakeSession) + monkeypatch.setattr(ba, "_browser_profile", lambda: None) + monkeypatch.setattr(ba, "_make_browser_llm", lambda: None) + monkeypatch.setattr(ba, "_has_page", lambda session: True) + monkeypatch.setattr(ba, "_screenshot_b64", nothing) + monkeypatch.setattr(ba, "_empty_required_fields", no_empty) + + async def run(): + queue: asyncio.Queue = asyncio.Queue() + queue.put_nowait("skip") + return [e async for e in ba.apply_with_browser("https://jobs.example.com/1", {}, "", queue, + answerer=answerer)] + return asyncio.run(run()) + + +def test_the_review_lists_what_was_answered_and_leads_with_what_only_you_can_answer(monkeypatch, model): + model(read={"Do you have the right to work in the UK?": fact("right_to_work_uk", answer="Yes"), + "Have you ever been convicted of a criminal offence?": fact()}) + + async def answerer(questions): + return await answers.answer_questions(questions, PERSONA, [], JOB) + events = _run_with_questions(monkeypatch, answerer, [ + ba.FormQuestion(question="Do you have the right to work in the UK?", options=["Yes", "No"]), + ba.FormQuestion(question="Have you ever been convicted of a criminal offence?", options=["Yes", "No"])]) + + review = next(e for e in events if e.get("interactive")) + assert [(q["question"][:20], q["source"]) for q in review["questions"]] == [ + ("Do you have the righ", "profile"), ("Have you ever been c", "needs_you")] + assert review["action"].startswith("Form filled, but 1 question only you can answer") + # The conviction question isn't listed twice; the other empty field still is. + assert review["notes"] == ("Questions only you can answer (left blank):\n" + "- Have you ever been convicted of a criminal offence?\n\n" + "Required fields still empty on the form:\n- Postcode\n\nFilled the form.") + + +def test_without_an_answerer_every_question_is_left_for_you(monkeypatch): + events = _run_with_questions(monkeypatch, None, [ba.FormQuestion(question="Notice period?")]) + review = next(e for e in events if e.get("interactive")) + assert review["questions"][0]["source"] == "needs_you" + + +def test_the_apply_stream_records_the_forms_questions_on_the_application(client, user, monkeypatch): + async def public(url): + return None + monkeypatch.setattr(net_guard, "assert_public_url", public) + asked = [{"question": "Notice period?", "options": [], "kind": "fact", "source": "profile", "answer": "2 weeks", + "because": "Your profile: Notice period = 2 weeks", "bank_id": "", "flags": []}] + seen = {} + + async def fake_apply_with_browser(*args, **kwargs): + seen["answerer"] = kwargs.get("answerer") + yield {"action": "Form filled", "screenshot": None, "blocked": True, "reason": "", "done": False, + "interactive": True, "questions": asked} + yield {"action": "Cancelled by user", "screenshot": None, "blocked": False, "reason": None, "done": True} + monkeypatch.setattr(agent, "apply_with_browser", fake_apply_with_browser) + + r = client.post("/apply", data={"job_id": "j", "job_title": "T", "company": "C", "job_description": "Ad", + "job_url": "https://jobs.example.com/1"}) + events = [json.loads(line[6:]) for line in r.text.splitlines() if line.startswith("data: ")] + assert next(e for e in events if e["type"] == "browser_blocked")["questions"] == asked + assert callable(seen["answerer"]) + assert tracker.list_applications(user)[0]["questions"] == asked diff --git a/api/tests/test_application_answers.py b/api/tests/test_application_answers.py index 479cc14..60bd981 100644 --- a/api/tests/test_application_answers.py +++ b/api/tests/test_application_answers.py @@ -6,31 +6,36 @@ import pytest import app.browser_agent as ba -from app import agent, net_guard, tracker +from app import agent, answers as answers_module, net_guard, tracker from app.models import is_present_end_date -UNSET_FACTS = ["Right to work", "Require visa sponsorship", "Has government ties", "Willing to relocate", - "Can work in-person", "Can start immediately", "Has reliable transportation", - "Needs accommodations", "Are you currently employed", "Remote/Hybrid preference", - "Salary expectations", "Currently holds another", "Previously held another"] +UNSET_FACTS = ["right_to_work_uk", "require_sponsorship", "has_government_ties", "relocation_willingness", + "can_work_in_person", "can_start_immediately", "has_reliable_transportation", + "needs_accommodations", "currently_employed", "remote_preference", + "salary_annual", "salary_hourly", "holds_other_nationality", "held_other_nationality"] def _answers(persona): - return dict(ba._known_answers(persona)) + """The persona's facts that answer_questions can answer from, by id.""" + return {fact.id: fact.value for fact in answers_module.profile_facts(persona)} def test_unanswered_facts_are_left_out_not_defaulted(): answers = _answers({"core_info": {"first_name": "Ada"}}) - for question in UNSET_FACTS: - assert not any(q.startswith(question) for q in answers), question + for fact in UNSET_FACTS: + assert fact not in answers, fact -def test_unanswered_facts_are_not_in_the_task_and_the_agent_is_told_not_to_guess(): - task = ba._build_task("https://x.example/job", {"core_info": {"first_name": "Ada"}}, "", "", "", "") - assert "work authorisation in the UK:" not in task - assert "Require visa sponsorship:" not in task - assert "Remote/Hybrid preference:" not in task - assert "do NOT guess" in task and "Needs your input" in task +def test_the_task_states_no_screening_facts_and_sends_every_question_to_answer_questions(): + persona = {"core_info": {"first_name": "Ada", "right_to_work_uk": True, "nationality": "Nigerian", + "visa_status": "Graduate visa"}, + "screening": {"why_this_role": "I like robots", "notice_period_canonical": "2 weeks"}, + "story_bank": [{"title": "Rescue", "scenario": "s", "action": "a", "result": "r"}]} + task = ba._build_task("https://x.example/job", persona, "", "", "", "") + for fact in ("work authorisation in the UK", "Nigerian", "Graduate visa", "I like robots", "2 weeks", "Rescue"): + assert fact not in task, fact + assert "call answer_questions first" in task and "Never answer one of these questions yourself" in task + assert "Do NOT guess" in task and "Needs your input" in task def test_agent_is_told_to_open_its_summary_with_what_needs_input(): @@ -44,26 +49,26 @@ def test_given_answers_are_stated_including_no(): "preferences": {"relocation_willingness": True, "can_start_immediately": False}, } answers = _answers(persona) - assert answers["Right to work / work authorisation in the UK"] == "No" - assert answers["Require visa sponsorship"] == "Yes" - assert answers["Has government ties"] == "No" - assert answers["Willing to relocate"] == "Yes" - assert answers["Can start immediately"] == "No" - assert "Needs accommodations" not in answers + assert answers["right_to_work_uk"] == "No" + assert answers["require_sponsorship"] == "Yes" + assert answers["has_government_ties"] == "No" + assert answers["relocation_willingness"] == "Yes" + assert answers["can_start_immediately"] == "No" + assert "needs_accommodations" not in answers def test_diversity_falls_back_to_prefer_not_to_say_and_maps_stored_values(): - assert _answers({})["Disability"] == "Prefer not to say" - assert _answers({"diversity": {"disability_status": "no"}})["Disability"] == "No" - assert _answers({"diversity": {"disability_status": "prefer_not_to_say"}})["Disability"] == "Prefer not to say" - assert _answers({})["Gender"] == "Prefer not to say" + assert _answers({})["disability"] == "Prefer not to say" + assert _answers({"diversity": {"disability_status": "no"}})["disability"] == "No" + assert _answers({"diversity": {"disability_status": "prefer_not_to_say"}})["disability"] == "Prefer not to say" + assert _answers({})["gender"] == "Prefer not to say" def test_currently_employed_means_a_current_role_not_any_past_one(): past = {"history": [{"employer": "Old Co", "title": "Intern", "is_current": False, "end_date": "2022"}]} current = {"history": [{"employer": "Now Co", "title": "Analyst", "is_current": True}]} - assert _answers(past)["Are you currently employed"] == "No" - assert _answers(current)["Are you currently employed"] == "Yes" + assert _answers(past)["currently_employed"] == "No" + assert _answers(current)["currently_employed"] == "Yes" @pytest.mark.parametrize("end_date", [ @@ -76,7 +81,7 @@ def test_a_present_end_date_counts_as_currently_employed(end_date): # How parsed CVs often store a current role: is_current left off, the CV's word in end_date. persona = {"history": [{"employer": "Old Co", "end_date": "2019"}, {"employer": "Now Co", "is_current": False, "end_date": end_date}]} - assert _answers(persona)["Are you currently employed"] == "Yes" + assert _answers(persona)["currently_employed"] == "Yes" @pytest.mark.parametrize("end_date", ["2021", "Mar 2019", "Nov 2020", "Nowhere Ltd", "Dated"]) @@ -87,17 +92,17 @@ def test_an_ended_role_is_not_mistaken_for_a_present_one(end_date): def test_currently_employed_is_no_only_when_every_role_has_really_ended(): ended = {"history": [{"employer": "A", "end_date": "2021"}, {"employer": "B", "end_date": "Mar 2019"}, {"employer": "C", "end_date": "03/2018"}, {"employer": "D", "end_date": "2016-07"}]} - assert _answers(ended)["Are you currently employed"] == "No" + assert _answers(ended)["currently_employed"] == "No" # A role with no end date may be the current one: don't claim either way. unclear = {"history": [{"employer": "A", "end_date": "2021"}, {"employer": "B", "end_date": ""}]} - assert "Are you currently employed" not in _answers(unclear) - assert "Are you currently employed" not in _answers({"history": [{"employer": "A"}]}) + assert "currently_employed" not in _answers(unclear) + assert "currently_employed" not in _answers({"history": [{"employer": "A"}]}) # Nor one whose end date has no year: it isn't a date we can read, so it may still mean "present". for end_date in ("Recently", "Summer", "Left", "Mar"): unread = {"history": [{"employer": "A", "end_date": "2021"}, {"employer": "B", "end_date": end_date}]} - assert "Are you currently employed" not in _answers(unread), end_date + assert "currently_employed" not in _answers(unread), end_date @pytest.mark.parametrize("end_date", [ @@ -106,18 +111,18 @@ def test_currently_employed_is_no_only_when_every_role_has_really_ended(): ]) def test_an_end_date_with_a_year_and_a_present_word_has_not_ended(end_date): persona = {"history": [{"employer": "A", "end_date": "2018"}, {"employer": "B", "end_date": end_date}]} - assert "Are you currently employed" not in _answers(persona) + assert "currently_employed" not in _answers(persona) @pytest.mark.parametrize("end_date", ["Nov 2022", "2016 - 2018", "Nowtel Ltd 2020", "Known as of 2021"]) def test_a_year_without_a_present_word_has_ended(end_date): persona = {"history": [{"employer": "A", "end_date": "2018"}, {"employer": "B", "end_date": end_date}]} - assert _answers(persona)["Are you currently employed"] == "No" + assert _answers(persona)["currently_employed"] == "No" def test_remote_preference_is_stated_only_when_the_user_picked_one(): - assert "Remote/Hybrid preference" not in _answers({"preferences": {"remote_preference": ""}}) - assert _answers({"preferences": {"remote_preference": "hybrid"}})["Remote/Hybrid preference"] == "hybrid" + assert "remote_preference" not in _answers({"preferences": {"remote_preference": ""}}) + assert _answers({"preferences": {"remote_preference": "hybrid"}})["remote_preference"] == "hybrid" def test_saving_a_profile_without_a_work_setting_leaves_it_unanswered_everywhere(client, user): @@ -130,12 +135,12 @@ def test_saving_a_profile_without_a_work_setting_leaves_it_unanswered_everywhere def test_salary_is_stated_only_as_the_candidate_gave_it(): # Unset: left out (see UNSET_FACTS) rather than "None", or a "Negotiable" they never chose. - assert not any(q.startswith("Salary") for q in _answers({"preferences": {"min_salary": None}})) + assert not any(q.startswith("salary") for q in _answers({"preferences": {"min_salary": None}})) answers = _answers({"screening": {"salary_canonical": "£32,000 - £36,000"}, "preferences": {"min_salary": 30000, "expected_hourly_rate": 18.5}}) - assert answers["Salary expectations (Annual)"] == "£32,000 - £36,000" # the written answer wins - assert answers["Salary expectations (Hourly)"] == "18.5" - assert _answers({"preferences": {"min_salary": 30000}})["Salary expectations (Annual)"] == "30000" + assert answers["salary_annual"] == "£32,000 - £36,000" # the written answer wins + assert answers["salary_hourly"] == "18.5" + assert _answers({"preferences": {"min_salary": 30000}})["salary_annual"] == "30000" def test_nationality_answers_are_stated_with_their_details(): @@ -143,11 +148,10 @@ def test_nationality_answers_are_stated_with_their_details(): "held_other_nationality": True, "previous_nationalities": "Ghanaian", "other_nationalities": "stale text from before"}} answers = _answers(persona) - assert answers["Currently holds another (dual) nationality or citizenship"] == "No" - assert answers["Previously held another nationality or citizenship"] == "Yes" - task = ba._build_task("https://x.example/job", persona, "", "", "", "") - assert "Nationality: Nigerian" in task and "Previously held nationalities / citizenships: Ghanaian" in task - assert "stale text" not in task # details only follow a "Yes" + assert answers["holds_other_nationality"] == "No" + assert answers["held_other_nationality"] == "Yes" + assert answers["nationality"] == "Nigerian" and answers["previous_nationalities"] == "Ghanaian" + assert "other_nationalities" not in answers # details only follow a "Yes" @pytest.mark.parametrize("core, kind, name", [ diff --git a/api/tests/test_browser_agent_secrets.py b/api/tests/test_browser_agent_secrets.py index 3c3d79b..0aef30b 100644 --- a/api/tests/test_browser_agent_secrets.py +++ b/api/tests/test_browser_agent_secrets.py @@ -290,12 +290,15 @@ def test_without_sensitive_data_actions_pass_through(scoped_tools): def test_both_agents_get_the_scoped_tools_and_the_secret(fake_browser, monkeypatch): - marker = object() - monkeypatch.setattr(ba, "_secret_scoped_tools", lambda: marker) + made = [] + scoped = ba._secret_scoped_tools + monkeypatch.setattr(ba, "_secret_scoped_tools", lambda: made.append(scoped()) or made[-1]) _apply(["submit"]) fill, submit = fake_browser assert submit["task"] == ba._SUBMIT_TASK - assert fill["tools"] is marker and submit["tools"] is marker + # The fill agent's set is the scoped one with answer_questions and fill_answer added to it. + assert fill["tools"] is made[0] and submit["tools"] is made[1] + assert {"answer_questions", "fill_answer"} <= set(fill["tools"].registry.registry.actions) assert fill["sensitive_data"] == submit["sensitive_data"] == {SCOPE: {"job_password": SECRET}} diff --git a/frontend/e2e/smoke.spec.ts b/frontend/e2e/smoke.spec.ts index 122abda..30501c4 100644 --- a/frontend/e2e/smoke.spec.ts +++ b/frontend/e2e/smoke.spec.ts @@ -261,3 +261,77 @@ test("the job workspace tailors a CV and checks it against the persona", async ( expect(checked[0]).toMatchObject({ type: "cv", doc: { name: "Sam Okafor" }, job: { title: job.title } }); api.expectAllMocked(); }); + +const answered = (question: string, source: string, answer: string, extra: object = {}) => ({ + question, options: [], kind: "fact", source, answer, because: "", bank_id: "", flags: [], ...extra, +}); + +test("the review shows where each answer came from and saves the one only you can answer", async ({ page }) => { + await signIn(page); + const posted: unknown[] = []; + const questions = [ + answered("Do you have the right to work in the UK?", "profile", "Yes", + { options: ["Yes", "No"], because: "Your profile: Right to work / work authorisation in the UK = Yes" }), + answered("Why do you want to join Company a1?", "drafted", "I want to build your booking services.", + { kind: "written", flags: ["Paragraph 1: Kafka isn't in your persona"] }), + answered("Have you ever been convicted of a criminal offence?", "needs_you", "", { options: ["Yes", "No"] }), + ]; + const api = await mockApi(page, { + "GET /api/persona": PERSONA, + "GET /api/applications": [app("a1", "Pending Review", { questions })], + "POST /api/answers": (request) => { + posted.push(request.postDataJSON()); + return { id: "b1", question: "q", answer: "No", kind: "fact", company: "", created_at: "", updated_at: "" }; + }, + }); + await page.goto("/review"); + const panel = page.getByRole("region", { name: "Questions on this form" }); + await expect(panel).toContainText("1 only you can answer"); + await expect(panel).toContainText("1 drafted, check it"); + await expect(panel).toContainText("Your profile: Right to work / work authorisation in the UK = Yes"); + await expect(panel).toContainText("Kafka isn't in your persona"); + + // The question nothing answered comes first, with the form's own options. + await panel.getByLabel("Have you ever been convicted of a criminal offence?").selectOption("No"); + await panel.getByRole("button", { name: "Save answer" }).click(); + await expect.poll(() => posted).toEqual([{ question: "Have you ever been convicted of a criminal offence?", + answer: "No", kind: "fact", company: "Company a1" }]); + await expect(panel).toContainText("it fills this in from your saved answers"); + api.expectAllMocked(); +}); + +test("saved answers can be edited and deleted from your persona", async ({ page }) => { + await signIn(page); + const saved = { id: "b1", question: "Do you hold a full UK driving licence?", answer: "No", kind: "fact", + company: "", created_at: "2026-10-16T10:00:00Z", updated_at: "2026-10-16T10:00:00Z" }; + const puts: unknown[] = []; + let deleted = false; + const api = await mockApi(page, { + ...SHELL, + "GET /api/answers": [saved], + "PUT /api/answers/b1": (request) => { + puts.push(request.postDataJSON()); + return { ...saved, answer: "Yes", updated_at: "2026-10-16T11:00:00Z" }; + }, + "DELETE /api/answers/b1": () => { + deleted = true; + return { deleted: "b1" }; + }, + }); + await page.goto("/profile"); + await page.getByRole("button", { name: /^Saved answers/ }).click(); + await expect(page).toHaveURL(/#answers$/); + await expect(page.getByText("Do you hold a full UK driving licence?")).toBeVisible(); + + await page.getByRole("button", { name: "Edit" }).click(); + await page.getByLabel("Do you hold a full UK driving licence?").fill("Yes"); + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect.poll(() => puts).toEqual([{ answer: "Yes" }]); + await expect(page.getByText("Yes", { exact: true })).toBeVisible(); + + await page.getByRole("button", { name: "Delete", exact: true }).click(); + await page.getByRole("button", { name: "Delete it" }).click(); + await expect.poll(() => deleted).toBe(true); + await expect(page.getByText("Nothing saved yet.", { exact: false })).toBeVisible(); + api.expectAllMocked(); +}); diff --git a/frontend/src/app/jobs/[id]/page.tsx b/frontend/src/app/jobs/[id]/page.tsx index 55c67e6..6e51217 100644 --- a/frontend/src/app/jobs/[id]/page.tsx +++ b/frontend/src/app/jobs/[id]/page.tsx @@ -10,6 +10,7 @@ import DocEditor, { CVDoc, LetterDoc, letterToText } from "@/components/DocEdito import ReviewLayout, { AgentNotes, type LogLine } from "@/components/ReviewLayout"; import JobDescription from "@/components/JobDescription"; import DocChecks from "@/components/DocChecks"; +import FormQuestions from "@/components/FormQuestions"; import Icon from "@/components/ui/Icon"; import TopBar, { Page } from "@/components/ui/TopBar"; import StepTracker from "@/components/ui/StepTracker"; @@ -24,7 +25,7 @@ import { discardJob, saveJob, trackedApplication } from "@/lib/actions"; import { load, useResource } from "@/lib/store"; import { fitVerdict, riskLevel, RISK } from "@/lib/status"; import { sponsorshipVerdict } from "@/lib/sponsorship"; -import type { Analysis, Application, Job } from "@/lib/types"; +import type { Analysis, Application, FormAnswer, Job } from "@/lib/types"; type CvChoice = "tailored" | "original"; @@ -106,6 +107,8 @@ export default function JobDetailPage() { const [browserInteractive, setBrowserInteractive] = useState(false); // The agent's summary ("Needs your input: …" first); sent once, kept for the whole review. const [browserNotes, setBrowserNotes] = useState(null); + // The form's questions, each with its answer and where it came from; sent with the notes. + const [browserQuestions, setBrowserQuestions] = useState([]); const [log, setLog] = useState([]); const [started, setStarted] = useState(false); const [applying, setApplying] = useState(false); @@ -280,6 +283,7 @@ export default function JobDetailPage() { } } if (typeof ev.notes === "string") setBrowserNotes(ev.notes); + if (Array.isArray(ev.questions)) setBrowserQuestions(ev.questions as FormAnswer[]); if (ev.type === "done") { setEnded({ submitted: ev.submitted === true }); setBrowserBlocked(false); @@ -295,6 +299,7 @@ export default function JobDetailPage() { setEnded(null); setApplyError(null); setBrowserNotes(null); + setBrowserQuestions([]); setBrowserScreenshot(null); setBrowserAction(""); setBrowserBlocked(false); @@ -314,6 +319,8 @@ export default function JobDetailPage() { form.append("job_title", job.title ?? ""); form.append("company", job.company ?? ""); form.append("location", job.location ?? ""); + // For written answers to the form's questions. + form.append("job_description", job.description ?? ""); form.append("cover_letter", letterDoc ? letterToText(letterDoc) : ""); form.append("tailored_cv", cvDoc ? JSON.stringify(cvDoc) : ""); form.append("cv_doc_id", cvChoice === "tailored" && effectiveCvDocId ? effectiveCvDocId : ""); @@ -739,6 +746,10 @@ export default function JobDetailPage() { {stepper} {review} + {review && ( + + )}
diff --git a/frontend/src/app/profile/page.tsx b/frontend/src/app/profile/page.tsx index c9f0fba..d976f68 100644 --- a/frontend/src/app/profile/page.tsx +++ b/frontend/src/app/profile/page.tsx @@ -8,6 +8,7 @@ import { buttonClass, Callout, Card, cx, Skeleton, Spinner } from "@/components/ import { AreaField, ChipInput, dateRange, EntryCard, monthLabel, SelectField, TextField, YesNoField, } from "@/components/persona/fields"; +import AnswerBank from "@/components/persona/AnswerBank"; import { load, mutate, requestJson } from "@/lib/store"; import { MIN_STORIES, personaChecks, personaStrength, type SectionKey } from "@/lib/persona"; import type { @@ -525,6 +526,8 @@ export default function ProfilePage() {
); + case "answers": + return ; case "prefs": return ( diff --git a/frontend/src/app/review/page.tsx b/frontend/src/app/review/page.tsx index 4d76311..429a186 100644 --- a/frontend/src/app/review/page.tsx +++ b/frontend/src/app/review/page.tsx @@ -4,6 +4,7 @@ import Link from "next/link"; import { Suspense, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; import { BrowserFrame } from "@/components/BrowserView"; +import FormQuestions from "@/components/FormQuestions"; import ReviewLayout from "@/components/ReviewLayout"; import Icon from "@/components/ui/Icon"; import { Page } from "@/components/ui/TopBar"; @@ -160,6 +161,8 @@ function ReviewQueue() { submit={{ onClick: () => { if (workspace) { workspace.remember(); router.push(workspace.href); } }, disabled: !workspace, label: "Submit from the workspace", hint: "Submitting needs the agent's browser open." }} /> + )} diff --git a/frontend/src/components/FormQuestions.tsx b/frontend/src/components/FormQuestions.tsx new file mode 100644 index 0000000..5d9328b --- /dev/null +++ b/frontend/src/components/FormQuestions.tsx @@ -0,0 +1,144 @@ +"use client"; + +import { useId, useState } from "react"; +import Icon from "./ui/Icon"; +import { buttonClass, Card, Chip, inputClass, Spinner, textareaClass, type ChipTone } from "./ui/kit"; +import { saveAnswer, SOURCE_LABEL } from "@/lib/answers"; +import type { AnswerSource, FormAnswer } from "@/lib/types"; + +const TONE: Record = { needs_you: "warn", drafted: "info", saved: "positive", profile: "neutral" }; +const ORDER: AnswerSource[] = ["needs_you", "drafted", "saved", "profile"]; + +function count(source: AnswerSource, n: number): string { + switch (source) { + case "needs_you": return `${n} only you can answer`; + case "drafted": return `${n} drafted, check ${n === 1 ? "it" : "them"}`; + case "saved": return `${n} saved`; + case "profile": return `${n} from your persona`; + } +} + +function Row({ q, company, savedNote }: { q: FormAnswer; company: string; savedNote: string }) { + const [value, setValue] = useState(q.answer); + const [editing, setEditing] = useState(q.source === "needs_you"); + const [state, setState] = useState<"idle" | "saving" | "saved">("idle"); + const [error, setError] = useState(null); + const labelId = useId(); + + const source: AnswerSource = state === "saved" ? "saved" : q.source; + const canSave = value.trim().length > 0 && state !== "saving"; + + const save = async () => { + setState("saving"); + setError(null); + try { + await saveAnswer(q.question, value.trim(), q.kind, company); + setState("saved"); + setEditing(false); + } catch (e) { + setState("idle"); + setError(e instanceof Error ? e.message : "Couldn't save the answer."); + } + }; + + return ( +
  • +
    + {q.question} + {SOURCE_LABEL[source]} +
    + + {editing ? ( +
    + {q.options.length > 0 ? ( + + ) : q.kind === "written" ? ( +