From 3868ced490c4111c313c208d69e932897a6b7ae9 Mon Sep 17 00:00:00 2001 From: Timothy Oluwatobi Ojebiyi Date: Fri, 16 Oct 2026 03:07:19 +0100 Subject: [PATCH 1/2] feat(api): critique-and-revise pass on letters, with a quality eval A critique pass reviews a tailored document against a recruiter's rubric for the ad and rewrites only what falls short. For letters the rubric is the swap test for 'why them', evidence mapped to the ad's requirements, the opening, the ending and the language. For CVs it is relevance, evidence, summary, projects and bullet order. A rewrite is kept only if the document check finds no new unsupported claim in it and it is at most 15% longer, so the page fit holds. The review (issues found, kept or not) goes out with the document. DOC_CRITIQUE picks the documents it runs on (cv, letter); letters only by default. evals/test_doc_quality_eval.py measures it: the graduate profile is tailored to six adverts with the real model, the critique runs on each draft, and a judge compares draft and revision blind in both orders. Over three runs the critiqued letter beat its draft every time (4-2, 5-1, 2-1 with 3 ties); CVs were mixed (1-3, 4-0, 2-0), so the CV critique stays off. The eval fails if a kept rewrite adds an unsupported claim or if letters stop winning. Tests turn the critique off by default (one more model call per document) and cover it explicitly: a rewrite that holds up is kept, one that invents a claim or runs long is dropped, and no issues leaves the document alone. --- .gitignore | 1 + api/app/agent.py | 176 ++++++++++++++++++++++++++--- api/evals/test_doc_quality_eval.py | 154 +++++++++++++++++++++++++ api/tests/conftest.py | 2 + api/tests/test_doc_check.py | 63 +++++++++++ 5 files changed, 379 insertions(+), 17 deletions(-) create mode 100644 api/evals/test_doc_quality_eval.py diff --git a/.gitignore b/.gitignore index c358679..15546e4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ frontend/playwright-report/ # Model evaluation output (api/evals) /api/evals/last_run.json +/api/evals/last_doc_run.json diff --git a/api/app/agent.py b/api/app/agent.py index 6269735..14f088a 100644 --- a/api/app/agent.py +++ b/api/app/agent.py @@ -610,36 +610,137 @@ def _issue_lines(findings: list[dict]) -> str: return "\n".join(lines) -async def _revise_cv_text(cv: dict, findings: list[dict], candidate: dict, job: dict) -> dict | None: - """ - One targeted pass over the CV's free text to fix what the document check flagged: claims the - candidate data doesn't back, and machine-sounding words. Returns the CV with the revised summary, - bullets and project text, or None when the reply doesn't keep the CV's shape. - """ - text = { +_CV_TEXT_SHAPE = ('{"summary": str, "experience": [{"title", "employer", "bullets": [str]}], ' + '"projects": [{"name", "description", "tech": [str]}]}') + + +def _cv_text(cv: dict) -> dict: + """The CV's free text (what a revision may change), with names to keep each part in place.""" + return { "summary": cv.get("summary", ""), "experience": [{"title": r.get("title", ""), "employer": r.get("employer", ""), "bullets": r.get("bullets", [])} for r in cv.get("experience") or []], "projects": [{"name": p.get("name", ""), "description": p.get("description", ""), "tech": p.get("tech", [])} for p in cv.get("projects") or []], } + + +def _word_count(value) -> int: + return sum(len(s.split()) for s in doc_check._flatten(value)) + + +async def _revise_cv_text(cv: dict, findings: list[dict], candidate: dict, job: dict) -> dict | None: + """ + One targeted pass over the CV's free text to fix what the document check flagged: claims the + candidate data doesn't back, and machine-sounding words. Returns the CV with the revised summary, + bullets and project text, or None when the reply doesn't keep the CV's shape. + """ data = await _llm_json( system=( "You check a tailored CV against the candidate's own data. Fix ONLY the listed issues and leave " "every other word as it is. A number, technology, employer or project the candidate data doesn't " "state must be removed or replaced by one it does state — never invent, estimate or round one. " "Keep each bullet's meaning and length. Return ONLY JSON in exactly the shape you were given: " - '{"summary": str, "experience": [{"title", "employer", "bullets": [str]}], ' - '"projects": [{"name", "description", "tech": [str]}]}, same roles and projects in the same order.' + f"{_CV_TEXT_SHAPE}, same roles and projects in the same order." ), - user=(f"ISSUES TO FIX:\n{_issue_lines(findings)}\n\nCV TEXT (JSON):\n{json.dumps(text, ensure_ascii=False)}\n\n" + user=(f"ISSUES TO FIX:\n{_issue_lines(findings)}\n\nCV TEXT (JSON):\n{json.dumps(_cv_text(cv), ensure_ascii=False)}\n\n" f"TARGET JOB: {job.get('title', '')} at {job.get('company', '')}\n\n" f"CANDIDATE DATA (JSON):\n{json.dumps(candidate, ensure_ascii=False)}"), max_tokens=2048, label="revise_cv", ) + return _merge_cv_text(cv, data) + + +_CV_RUBRIC = ( + "1. RELEVANCE: the ad's three most important requirements are visibly answered in the summary and in the " + "first bullet of the most relevant role or project, in the ad's own words where the candidate data supports " + "them. 2. EVIDENCE: every bullet is a strong verb + what was built or done + a concrete result or scope that " + "the candidate data states; no generic claims ('collaborated with cross-functional teams', 'worked on " + "various projects'). 3. SUMMARY: a specific specialism and domain plus one concrete strength; no cliché " + "openers or closing filler, no pronouns. 4. PROJECTS: what it does, how it works, and a real result, framed " + "for this job's domain. 5. ORDER: the most job-relevant bullet comes first in each role." +) + +_LETTER_RUBRIC = ( + "1. SPECIFIC TO THEM: the 'why them' paragraph passes the swap test — if another employer's name could " + "replace theirs and it still reads fine, it fails. 2. EVIDENCE: two or three requirements named in the ad " + "are each proved by a named project or role from the candidate data, with a real result. 3. OPENING: a " + "specific hook tied to this employer, never 'I am writing to apply'. 4. ENDING: short and concrete; no perks " + "or salary. 5. LANGUAGE: active, confident, no hedging ('I believe', 'I feel'), no clichés or filler." +) + + +async def _critique_cv_text(cv: dict, candidate: dict, job: dict) -> tuple[dict | None, list[str]]: + """ + A critique-and-revise pass on writing quality: the model reviews the CV text against _CV_RUBRIC + and the ad, lists what falls short, and rewrites only those parts. Returns (revised CV or None, + issues found). The caller keeps the revision only if it passes the document check. + """ + data = await _llm_json( + system=( + "You are a demanding recruiter reviewing a tailored CV for one job. Judge the CV text against this " + f"rubric: {_CV_RUBRIC} " + "List each real shortfall in 'issues' (short, specific: where and what). Then rewrite ONLY the parts " + "with an issue in 'revised'; copy everything else exactly. STRICT: use only facts the candidate data " + "states — never add a number, technology, employer or project it doesn't, and never estimate one. Keep " + "names, the same roles and projects in the same order, and about the same length (the CV must still " + "fit one page). If nothing falls short, return an empty 'issues' list. Return ONLY JSON: " + f'{{"issues": [str], "revised": {_CV_TEXT_SHAPE}}}.' + ), + user=(f"TARGET JOB: {job.get('title', '')} at {job.get('company', '')}\n" + f"JOB DESCRIPTION:\n{job.get('description', '')[:2500]}\n\n" + f"CV TEXT (JSON):\n{json.dumps(_cv_text(cv), ensure_ascii=False)}\n\n" + f"CANDIDATE DATA (JSON):\n{json.dumps(candidate, ensure_ascii=False)}"), + max_tokens=3000, + label="critique_cv", + ) + if not isinstance(data, dict): + return None, [] + issues = [str(i).strip() for i in data.get("issues") or [] if str(i).strip()] + if not issues: + return None, [] + return _merge_cv_text(cv, data.get("revised")), issues + + +async def _critique_letter_text(letter: dict, persona: dict, job: dict, research: str) -> tuple[dict | None, list[str]]: + """The cover-letter counterpart of _critique_cv_text, judged against _LETTER_RUBRIC.""" + data = await _llm_json( + system=( + "You are a demanding recruiter reviewing a cover letter for one job. Judge it against this rubric: " + f"{_LETTER_RUBRIC} " + "List each real shortfall in 'issues' (short, specific: which paragraph and what). Then rewrite ONLY " + "the paragraphs with an issue; copy the others exactly. STRICT: use only facts from the candidate " + "data, the job ad and the company research — never add a number or technology none of them states. " + "Keep the same number of paragraphs and about the same length (one page). If nothing falls short, " + 'return an empty \'issues\' list. Return ONLY JSON: {"issues": [str], "paragraphs": [str]}.' + ), + user=(f"JOB: {job.get('title', '')} at {job.get('company', '')}\n" + f"DESCRIPTION: {job.get('description', '')[:2500]}\n\n" + f"LETTER (JSON):\n{json.dumps({'paragraphs': letter.get('paragraphs', [])}, ensure_ascii=False)}\n\n" + f"CANDIDATE PERSONA (JSON): {json.dumps(_llm_persona(persona), ensure_ascii=False)}" + + (f"\n\nCOMPANY RESEARCH:\n{research}" if research else "")), + max_tokens=1500, + label="critique_letter", + ) + if not isinstance(data, dict): + return None, [] + issues = [str(i).strip() for i in data.get("issues") or [] if str(i).strip()] + paragraphs = data.get("paragraphs") + if not issues: + return None, [] + if (not isinstance(paragraphs, list) or len(paragraphs) != len(letter.get("paragraphs") or []) + or not all(isinstance(p, str) and p.strip() for p in paragraphs)): + return None, issues + return {**letter, "paragraphs": [p.strip() for p in paragraphs]}, issues + + +def _merge_cv_text(cv: dict, data) -> dict | None: + """The CV with a revision's summary, bullets and project text, or None when the revision doesn't + keep the CV's shape (same roles and projects, in the same order).""" if not isinstance(data, dict): return None + text = _cv_text(cv) roles, projects = data.get("experience"), data.get("projects") if (not isinstance(roles, list) or len(roles) != len(text["experience"]) or not isinstance(projects, list) or len(projects) != len(text["projects"])): @@ -684,6 +785,25 @@ async def _revise_letter_text(letter: dict, findings: list[dict], persona: dict, return {**letter, "paragraphs": [p.strip() for p in paragraphs]} +# Which documents get the critique-and-revise pass on writing quality: DOC_CRITIQUE is a comma-separated +# list of "cv" and "letter" ("0" or empty for none). Letters only by default. On the quality eval +# (evals/test_doc_quality_eval.py, three runs of six adverts) the critiqued letter beat its draft every run +# (4-2, 5-1, then 2-1 with 3 ties when judged in both orders); CVs were mixed (1-3, 4-0, 2-0), so for CVs +# it isn't yet worth the extra model call. +_DOC_CRITIQUE = {k.strip() for k in os.environ.get("DOC_CRITIQUE", "letter").split(",")} & {"cv", "letter"} +_CRITIQUE_GROWTH = 1.15 # a rewrite may run this much longer before it would crowd the page + + +def _critique_holds(doc_type: str, before: dict, after: dict, persona: dict, job: dict, research: str = "") -> bool: + """A quality rewrite is kept only if the document check finds nothing new in it and it hasn't grown + past _CRITIQUE_GROWTH.""" + def found(doc): + return doc_check.check_facts(doc_type, doc, persona, job, research) + doc_check.check_style(doc_type, doc) + text = (lambda d: d.get("paragraphs")) if doc_type == "cover_letter" else _cv_text + return (_finding_keys(found(after)) <= _finding_keys(found(before)) + and _word_count(text(after)) <= _CRITIQUE_GROWTH * max(1, _word_count(text(before)))) + + def _better(before: list[dict], after: list[dict]) -> bool: """A revision is kept only when it clears findings and introduces none of its own.""" return len(after) < len(before) and _finding_keys(after) <= _finding_keys(before) @@ -1155,6 +1275,16 @@ async def execute_tool(name: str, tool_input: dict) -> dict: ) letter["signoff"] = "Yours faithfully," if _generic_salutation else "Yours sincerely," + # Critique and revise against the rubric; kept only if it adds no unsupported claim and + # stays about the same length. + review: dict = {"issues": [], "kept": False} + if data and "letter" in _DOC_CRITIQUE: + revised, review["issues"] = await _critique_letter_text(letter, persona, job, company_research) + if revised is not None: + review["kept"] = _critique_holds("cover_letter", letter, revised, persona, job, company_research) + if review["kept"]: + letter = revised + # Numbers and tools the persona, the ad and the company research don't state, and # machine-sounding words: one targeted revision, kept only if it clears them. revision = "fallback" @@ -1175,9 +1305,10 @@ async def execute_tool(name: str, tool_input: dict) -> dict: letter = revised (logger.warning if data is None else logger.info)( - "cover_letter: llm_used=%s, paragraphs=%d, archetype=%s, level=%s, revision=%s | job=%r", - data is not None, len(letter["paragraphs"]), archetype, level, revision, - (job.get("title") or "")[:60], + "cover_letter: llm_used=%s, paragraphs=%d, archetype=%s, level=%s, critique=%d issues %s, revision=%s " + "| job=%r", + data is not None, len(letter["paragraphs"]), archetype, level, len(review["issues"]), + "kept" if review["kept"] else "not kept", revision, (job.get("title") or "")[:60], ) return { @@ -1187,6 +1318,7 @@ async def execute_tool(name: str, tool_input: dict) -> dict: "candidate_name": full_name, "letter": letter, "archetype": archetype, + "review": review, } elif name == "generate_skill_roadmap": @@ -1375,6 +1507,15 @@ async def execute_tool(name: str, tool_input: dict) -> dict: added_skills: list[str] = [] if chosen: cv["skills"], added_skills = _top_up_skills(cv["skills"], base_cv["skills"], cv, job) + # Critique and revise against the rubric; kept only if it adds no unsupported claim and + # stays about the same length (the page fit is measured on the final text). + review: dict = {"issues": [], "kept": False} + if not used_fallback and "cv" in _DOC_CRITIQUE: + revised, review["issues"] = await _critique_cv_text(cv, llm_cv, job) + if revised is not None: + review["kept"] = _critique_holds("cv", cv, revised, persona, job) + if review["kept"]: + cv = revised # Check the text against the candidate's data; one targeted revision when it flags something. revision = "fallback" if not used_fallback: @@ -1396,12 +1537,13 @@ async def execute_tool(name: str, tool_input: dict) -> dict: # same projects recur across roles (the model is meant to tailor the selection). final_proj_names = [(p.get("name") or "")[:40] for p in (cv.get("projects") or [])][:5] (logger.warning if used_fallback else logger.info)( - "tailored_cv: fallback=%s, level=%s, skills chosen=%s, added=%s, revision=%s, projects base=%d model=%d " - "final=%d %s | job=%r", - used_fallback, level, chosen is not None, added_skills, revision, base_proj_n, model_proj_n, + "tailored_cv: fallback=%s, level=%s, skills chosen=%s, added=%s, critique=%d issues %s, revision=%s, " + "projects base=%d model=%d final=%d %s | job=%r", + used_fallback, level, chosen is not None, added_skills, len(review["issues"]), + "kept" if review["kept"] else "not kept", revision, base_proj_n, model_proj_n, len(cv.get("projects") or []), final_proj_names, (job.get("title") or "")[:60], ) - return {"cv": cv, "template_id": template_id} + return {"cv": cv, "template_id": template_id, "review": review} return {"error": f"Unknown tool: {name}"} diff --git a/api/evals/test_doc_quality_eval.py b/api/evals/test_doc_quality_eval.py new file mode 100644 index 0000000..1af1b36 --- /dev/null +++ b/api/evals/test_doc_quality_eval.py @@ -0,0 +1,154 @@ +""" +Does the critique-and-revise pass make tailored CVs and cover letters better? + +For each advert below, the graduate profile from tests/test_documents.py is tailored with the real +model and the critique off (the draft). The critique then runs on the draft as it does in +production, including the guard that drops a rewrite with an unsupported claim or that runs long. +Where it changed something, a judge sees the draft and the revision blind, in both orders (a version +wins only when both agree) and says which better serves the advert. Every run writes evals/last_doc_run.json. + +The bar: no kept revision may carry a claim the document check flags that the draft didn't, and for +cover letters (which get the critique by default) the revision must win more often than it loses. +CVs are measured only: across the first three runs they were mixed (draft 3-1, then revision 4-0 and +2-0), so the CV critique is off by default. +""" +import asyncio +import json +import pathlib + +import pytest + +from app import agent, doc_check +from test_documents import GRADUATE + +HERE = pathlib.Path(__file__).parent + +ADS = [ + {"title": "Graduate Backend Engineer", "company": "Harbourline", "location": "London", + "description": "Harbourline builds the booking platform behind 400 ferry routes. As a graduate backend engineer " + "you will design and ship Python services (FastAPI) backed by PostgreSQL, write tests with " + "pytest, and run them in Docker on AWS. You will own features end to end, from API design to " + "monitoring in production. We look for: a degree in a STEM subject or equivalent experience; " + "solid Python; SQL; an understanding of REST API design; curiosity about distributed systems."}, + {"title": "Junior Data Engineer", "company": "Meridian Health Analytics", "location": "Bristol", + "description": "Join a small team turning hospital operations data into dashboards that clinicians use daily. " + "You will build ETL pipelines in Python and SQL, model data in PostgreSQL, schedule jobs with " + "Airflow and keep data quality high with automated tests. Nice to have: pandas, dbt, AWS. " + "You care about getting numbers right and explaining them to non-technical colleagues."}, + {"title": "Graduate Machine Learning Engineer", "company": "Signalbox Rail", "location": "Manchester", + "description": "Signalbox predicts disruption on UK rail networks. You will train and evaluate forecasting " + "models in Python with PyTorch and scikit-learn, build data pipelines for live feeds, and help " + "deploy models behind APIs. We value rigorous evaluation, clear write-ups and pragmatic " + "engineering. Requirements: Python, machine learning fundamentals, SQL, Git."}, + {"title": "Graduate Frontend Developer", "company": "Larder", "location": "Remote, UK", + "description": "Larder helps households waste less food. You will build features in React and Next.js with " + "TypeScript, write component and end-to-end tests (Jest, Playwright), and work closely with " + "design. You will ship to thousands of users weekly. We want: TypeScript, React, attention to " + "accessibility and performance, and a portfolio of things you have built."}, + {"title": "Graduate Software Engineer, Trading Systems", "company": "Quayside Capital", "location": "London", + "description": "Write low-latency, well-tested software for our trading platform in Java and Python. You will " + "work on market data services, use data structures and algorithms to make them fast, and pair " + "with traders to understand requirements. Strong programming fundamentals, a quantitative " + "mindset and a 2:1 or above in Computer Science, Maths or Physics."}, + {"title": "Graduate DevOps Engineer", "company": "Cloudrail", "location": "Leeds", + "description": "Help our platform team automate everything. You will build CI/CD pipelines with GitHub " + "Actions, containerise services with Docker, manage AWS infrastructure and write Python " + "tooling. Kubernetes and Terraform experience is a bonus. You enjoy making other engineers " + "faster and systems more reliable."}, +] + +JUDGE = ( + "You are an experienced UK graduate recruiter. You see one job advert and two versions (A and B) of the same " + "candidate's {kind}, written from the same facts. Decide which version better serves THIS advert: it answers " + "the advert's main requirements visibly and in the advert's terms, proves claims with concrete specifics and " + "results, reads as written by a person (no clichés, filler or hedging), and is concise. Ignore formatting. " + 'If they are equally good, say tie. Return ONLY JSON: {{"better": "A" | "B" | "tie", "reason": str}}.' +) + + +def _candidate(job): + base = agent._persona_to_cvdoc(GRADUATE) + return {**base, "contact": {}, "projects": agent._rank_projects_by_relevance(job, agent._persona_projects_for_llm(GRADUATE))} + + +async def _judge_once(kind: str, job: dict, draft, revised, revised_first: bool) -> tuple[str, str]: + a, b = (revised, draft) if revised_first else (draft, revised) + verdict = await agent._llm_json( + system=JUDGE.format(kind=kind), + user=f"ADVERT: {job['title']} at {job['company']}\n{job['description']}\n\n" + f"VERSION A:\n{json.dumps(a, ensure_ascii=False)}\n\nVERSION B:\n{json.dumps(b, ensure_ascii=False)}", + max_tokens=600, label="eval_judge") + pick, reason = (verdict or {}).get("better", "tie"), str((verdict or {}).get("reason", "")) + if pick == "tie" or pick not in ("A", "B"): + return "tie", reason + return ("revised" if (pick == "A") == revised_first else "draft"), reason + + +async def _judge(kind: str, job: dict, draft, revised) -> tuple[str, list[str]]: + """Judged in both orders; a version wins only when both say so (position bias cancels out).""" + (first, r1), (second, r2) = await asyncio.gather(_judge_once(kind, job, draft, revised, True), + _judge_once(kind, job, draft, revised, False)) + return (first if first == second else "tie"), [r1, r2] + + +async def _one(i: int, job: dict) -> dict: + out = {"job": f"{job['title']} @ {job['company']}"} + cv = (await agent.execute_tool("generate_tailored_cv", {"job": job, "persona": GRADUATE}))["cv"] + letter = (await agent.execute_tool("generate_cover_letter", {"job": job, "persona": GRADUATE}))["letter"] + + revised_cv, cv_issues = await agent._critique_cv_text(cv, _candidate(job), job) + cv_kept = revised_cv is not None and agent._critique_holds("cv", cv, revised_cv, GRADUATE, job) + revised_letter, letter_issues = await agent._critique_letter_text(letter, GRADUATE, job, "") + letter_kept = revised_letter is not None and agent._critique_holds("cover_letter", letter, revised_letter, GRADUATE, job) + + for kind, draft, revised, issues, kept, text in ( + ("CV", cv, revised_cv, cv_issues, cv_kept, agent._cv_text), + ("cover letter", letter, revised_letter, letter_issues, letter_kept, lambda d: d.get("paragraphs")), + ): + key = "cv" if kind == "CV" else "letter" + new_flags = [] + if revised is not None: + before = doc_check.check_facts("cv" if key == "cv" else "cover_letter", draft, GRADUATE, job) + after = doc_check.check_facts("cv" if key == "cv" else "cover_letter", revised, GRADUATE, job) + new_flags = sorted(agent._finding_keys(after) - agent._finding_keys(before)) + result = {"issues": issues, "rewrote": revised is not None, "kept": kept, "new_flags": [list(f) for f in new_flags], + "draft": text(draft), "revised": text(revised) if revised is not None else None} + if kept: + result["verdict"], result["reasons"] = await _judge(kind, job, text(draft), text(revised)) + out[key] = result + return out + + +@pytest.fixture +def no_research(monkeypatch): + async def none(*args, **kwargs): + return "" + monkeypatch.setattr(agent, "_research_company", none) + monkeypatch.setattr(agent, "_DOC_CRITIQUE", set()) # drafts without the critique; it runs separately + + +def test_critique_improves_documents(no_research): + async def run(): + sem = asyncio.Semaphore(3) + + async def guarded(i, job): + async with sem: + return await _one(i, job) + return await asyncio.gather(*(guarded(i, j) for i, j in enumerate(ADS))) + + results = asyncio.run(run()) + (HERE / "last_doc_run.json").write_text(json.dumps(results, ensure_ascii=False, indent=1), encoding="utf-8") + + score = {} + for key in ("cv", "letter"): + verdicts = [r[key]["verdict"] for r in results if "verdict" in r[key]] + score[key] = (verdicts.count("revised"), verdicts.count("draft")) + print(f"{key}: issues found {sum(bool(r[key]['issues']) for r in results)}/{len(results)}, " + f"rewrites kept {sum(r[key]['kept'] for r in results)}, revised won {score[key][0]}, " + f"lost {score[key][1]}, tied {verdicts.count('tie')}") + # The guard must have dropped any rewrite with a new unsupported claim. + assert not any(r[key]["kept"] and r[key]["new_flags"] for r in results) + # Letters get the critique by default (agent._DOC_CRITIQUE), so they must gain from it; CVs are measured + # so a change to the CV critique can be judged before it's turned on. + wins, losses = score["letter"] + assert wins > losses, f"letters: the critique didn't beat its drafts ({wins} wins, {losses} losses)" diff --git a/api/tests/conftest.py b/api/tests/conftest.py index a2a8069..dad5995 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -71,6 +71,8 @@ def do_POST(self): "GROQ_API_KEY": "test-key", "OLLAMA_HOST": f"http://127.0.0.1:{_ollama.server_address[1]}", "ANONYMIZED_TELEMETRY": "false", + # The critique pass is one more model call per document; tests that cover it turn it on. + "DOC_CRITIQUE": "0", }) diff --git a/api/tests/test_doc_check.py b/api/tests/test_doc_check.py index 5fedb1a..fa216b1 100644 --- a/api/tests/test_doc_check.py +++ b/api/tests/test_doc_check.py @@ -223,3 +223,66 @@ def test_a_letter_with_a_flagged_word_is_revised(replies): json.dumps({"paragraphs": ["I use FastAPI daily.", "Your team of 40 engineers ships daily."]})] letter = asyncio.run(agent.execute_tool("generate_cover_letter", {"job": AD, "persona": GRADUATE}))["letter"] assert letter["paragraphs"] == ["I use FastAPI daily.", "Your team of 40 engineers ships daily."] + + +# ── Critique and revise (writing quality) ───────────────────────────────────────────────────── + +@pytest.fixture +def critique(monkeypatch, replies): + monkeypatch.setattr(agent, "_DOC_CRITIQUE", {"cv", "letter"}) + return replies + + +GOOD_BULLET = "Built a FastAPI service ingesting 40k readings a day into PostgreSQL." + + +def _critique_reply(summary: str, bullet: str = GOOD_BULLET, issues=("Summary misses the ad's FastAPI focus",)) -> str: + text = json.loads(_text_reply(bullet)) + text["summary"] = summary + return json.dumps({"issues": list(issues), "revised": text}) + + +def _generate(kind: str): + tool = "generate_tailored_cv" if kind == "cv" else "generate_cover_letter" + return asyncio.run(agent.execute_tool(tool, {"job": AD, "persona": GRADUATE})) + + +def test_a_critique_rewrite_that_holds_up_is_kept(critique): + better = "Computer Science graduate building Python and FastAPI services on PostgreSQL." + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), _critique_reply(better)] + out = _generate("cv") + assert out["cv"]["summary"] == better + assert out["review"] == {"issues": ["Summary misses the ad's FastAPI focus"], "kept": True} + assert "demanding recruiter" in critique.prompts[1] and "RELEVANCE" in critique.prompts[1] + + +def test_a_critique_rewrite_that_invents_a_claim_is_dropped(critique): + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), + _critique_reply("Graduate who cut latency by 35% with FastAPI.")] + out = _generate("cv") + assert out["cv"]["summary"] == "Computer Science graduate who builds Python and FastAPI services." + assert out["review"]["kept"] is False + + +def test_a_critique_rewrite_that_runs_long_is_dropped(critique): + long_summary = "Computer Science graduate who builds Python and FastAPI services. " * 6 + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), _critique_reply(long_summary)] + assert _generate("cv")["review"]["kept"] is False + + +def test_no_issues_leaves_the_cv_alone(critique): + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), json.dumps({"issues": [], "revised": {}})] + out = _generate("cv") + assert out["cv"]["experience"][0]["bullets"] == [GOOD_BULLET] + assert out["review"] == {"issues": [], "kept": False} + + +def test_the_letter_critique_rewrites_the_flagged_paragraph(critique): + first = {"salutation": "Dear Hiring Manager,", "signoff": "Yours faithfully,", "headline": "Graduate", + "paragraphs": ["I am writing to apply for this role.", "I build FastAPI services on PostgreSQL."]} + critique.queue = [json.dumps(first), json.dumps({ + "issues": ["Opening is generic"], + "paragraphs": ["Northwind's API platform is where I want to build.", "I build FastAPI services on PostgreSQL."]})] + out = _generate("letter") + assert out["letter"]["paragraphs"][0] == "Northwind's API platform is where I want to build." + assert out["review"] == {"issues": ["Opening is generic"], "kept": True} From 2c313a5144032b1d1513b9198c5473045bad9e97 Mon Sep 17 00:00:00 2001 From: Timothy Oluwatobi Ojebiyi Date: Fri, 16 Oct 2026 03:07:20 +0100 Subject: [PATCH 2/2] feat(frontend): show what the review fixed in the checks strip When the critique's rewrite was kept, the Checks strip says how many things the review fixed for this ad, and its details list them. --- frontend/e2e/smoke.spec.ts | 5 ++++- frontend/src/app/jobs/[id]/page.tsx | 24 ++++++++++++++++++++---- frontend/src/components/DocChecks.tsx | 20 +++++++++++++++++++- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/frontend/e2e/smoke.spec.ts b/frontend/e2e/smoke.spec.ts index 482f0cc..122abda 100644 --- a/frontend/e2e/smoke.spec.ts +++ b/frontend/e2e/smoke.spec.ts @@ -233,7 +233,8 @@ test("the job workspace tailors a CV and checks it against the persona", async ( "GET /api/jobs/1/sponsorship": { sponsorship: "offered", evidence: "", clearance: "", licensed_sponsor: false, sponsor_name: "", mark: "" }, "POST /api/jobs/analyze": { sse: [{ type: "job_fit", ranked_jobs: [{ fit_score: 84, level_match: "strong", matched_skills: ["Python"], skill_gaps: ["Kubernetes"], reasoning: "Strong backend overlap." }] }] }, - "POST /api/jobs/tailored-cv": { sse: [{ type: "tailored_cv", cv }] }, + "POST /api/jobs/tailored-cv": { sse: [{ type: "tailored_cv", cv, + review: { issues: ["Summary didn't name FastAPI"], kept: true } }] }, "POST /api/documents/preview": { html: "CV" }, "POST /api/documents/check": (request) => { checked.push(request.postDataJSON()); @@ -252,7 +253,9 @@ test("the job workspace tailors a CV and checks it against the persona", async ( const checks = page.getByRole("button", { name: /^Checks/ }); await expect(checks).toContainText("1 thing to check"); await expect(checks).toContainText("Keywords 2 of 3 you have"); + await expect(checks).toContainText("Reviewed for this ad: 1 fix"); await checks.click(); + await expect(page.getByText("Summary didn't name FastAPI")).toBeVisible(); await expect(page.getByText("“35%” isn't in your persona")).toBeVisible(); await expect(page.getByText("Docker", { exact: true })).toBeVisible(); expect(checked[0]).toMatchObject({ type: "cv", doc: { name: "Sam Okafor" }, job: { title: job.title } }); diff --git a/frontend/src/app/jobs/[id]/page.tsx b/frontend/src/app/jobs/[id]/page.tsx index 3580db9..55c67e6 100644 --- a/frontend/src/app/jobs/[id]/page.tsx +++ b/frontend/src/app/jobs/[id]/page.tsx @@ -51,6 +51,12 @@ async function downloadPdf(docId: string, filename: string) { URL.revokeObjectURL(url); } +/** The issues a kept critique rewrite fixed, from the generation event's `review`. */ +function keptIssues(review: unknown): string[] { + const r = review as { issues?: unknown; kept?: unknown } | undefined; + return r?.kept === true && Array.isArray(r.issues) ? r.issues.filter((i): i is string => typeof i === "string") : []; +} + const LEVEL_MATCH: Record = { strong: "strong", ok: "reasonable", reach: "a stretch" }; const EXPOSURE_NOTE = { @@ -82,6 +88,8 @@ export default function JobDetailPage() { const [cvError, setCvError] = useState(null); const [letterError, setLetterError] = useState(null); const [cvChoice, setCvChoice] = useState("tailored"); + // What the critique-and-revise pass found and fixed, when its rewrite was kept. + const [reviewed, setReviewed] = useState<{ cv: string[]; letter: string[] }>({ cv: [], letter: [] }); // PDF state const [cvDocId, setCvDocId] = useState(null); @@ -179,7 +187,11 @@ export default function JobDetailPage() { let got = false; try { await postStream("/api/jobs/tailored-cv", { job }, (ev) => { - if (ev.type === "tailored_cv" && ev.cv) { setCvDoc(ev.cv as CVDoc); got = true; } + if (ev.type === "tailored_cv" && ev.cv) { + setCvDoc(ev.cv as CVDoc); + setReviewed((r) => ({ ...r, cv: keptIssues(ev.review) })); + got = true; + } }); if (!got) setCvError("The tailored CV didn't come through."); } catch (e) { @@ -197,7 +209,11 @@ export default function JobDetailPage() { let got = false; try { await postStream("/api/jobs/cover-letter", { job }, (ev) => { - if (ev.type === "cover_letter" && ev.letter) { setLetterDoc(ev.letter as LetterDoc); got = true; } + if (ev.type === "cover_letter" && ev.letter) { + setLetterDoc(ev.letter as LetterDoc); + setReviewed((r) => ({ ...r, letter: keptIssues(ev.review) })); + got = true; + } }); if (!got) setLetterError("The cover letter didn't come through."); } catch (e) { @@ -534,7 +550,7 @@ export default function JobDetailPage() { {docToolbar("cv")} - {cvDoc && !streamingCv && } + {cvDoc && !streamingCv && } {streamingCv ? docSkeleton("Tailoring your CV for this role…") : cvDoc ? ( { setCvDoc(c); setCvDocId(null); }} /> ) : ( @@ -562,7 +578,7 @@ export default function JobDetailPage() { {docToolbar("cover_letter")} - {letterDoc && !streamingLetter && } + {letterDoc && !streamingLetter && } {streamingLetter ? docSkeleton("Writing your cover letter…") : letterDoc ? ( { setLetterDoc(l); setLetterDocId(null); }} /> diff --git a/frontend/src/components/DocChecks.tsx b/frontend/src/components/DocChecks.tsx index f59ee1d..1dbe716 100644 --- a/frontend/src/components/DocChecks.tsx +++ b/frontend/src/components/DocChecks.tsx @@ -30,8 +30,10 @@ function Pill({ tone, children }: { tone: "ok" | "warn" | "neutral"; children: R * claims your persona doesn't back, machine-sounding words, and the PDF read back as an applicant * tracking system reads it. Re-runs a moment after each edit. */ -export default function DocChecks({ kind, doc, job, templateId = "classic" }: { +export default function DocChecks({ kind, doc, job, templateId = "classic", reviewed = [] }: { kind: "cv" | "cover_letter"; doc: object; job: Job; templateId?: string; + /** What the review against the ad found and rewrote while the document was written. */ + reviewed?: string[]; }) { const [report, setReport] = useState(null); const [checking, setChecking] = useState(false); @@ -84,6 +86,9 @@ export default function DocChecks({ kind, doc, job, templateId = "classic" }: {