diff --git a/.gitignore b/.gitignore index c358679..15546e4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ frontend/playwright-report/ # Model evaluation output (api/evals) /api/evals/last_run.json +/api/evals/last_doc_run.json diff --git a/api/app/agent.py b/api/app/agent.py index 6269735..14f088a 100644 --- a/api/app/agent.py +++ b/api/app/agent.py @@ -610,36 +610,137 @@ def _issue_lines(findings: list[dict]) -> str: return "\n".join(lines) -async def _revise_cv_text(cv: dict, findings: list[dict], candidate: dict, job: dict) -> dict | None: - """ - One targeted pass over the CV's free text to fix what the document check flagged: claims the - candidate data doesn't back, and machine-sounding words. Returns the CV with the revised summary, - bullets and project text, or None when the reply doesn't keep the CV's shape. - """ - text = { +_CV_TEXT_SHAPE = ('{"summary": str, "experience": [{"title", "employer", "bullets": [str]}], ' + '"projects": [{"name", "description", "tech": [str]}]}') + + +def _cv_text(cv: dict) -> dict: + """The CV's free text (what a revision may change), with names to keep each part in place.""" + return { "summary": cv.get("summary", ""), "experience": [{"title": r.get("title", ""), "employer": r.get("employer", ""), "bullets": r.get("bullets", [])} for r in cv.get("experience") or []], "projects": [{"name": p.get("name", ""), "description": p.get("description", ""), "tech": p.get("tech", [])} for p in cv.get("projects") or []], } + + +def _word_count(value) -> int: + return sum(len(s.split()) for s in doc_check._flatten(value)) + + +async def _revise_cv_text(cv: dict, findings: list[dict], candidate: dict, job: dict) -> dict | None: + """ + One targeted pass over the CV's free text to fix what the document check flagged: claims the + candidate data doesn't back, and machine-sounding words. Returns the CV with the revised summary, + bullets and project text, or None when the reply doesn't keep the CV's shape. + """ data = await _llm_json( system=( "You check a tailored CV against the candidate's own data. Fix ONLY the listed issues and leave " "every other word as it is. A number, technology, employer or project the candidate data doesn't " "state must be removed or replaced by one it does state — never invent, estimate or round one. " "Keep each bullet's meaning and length. Return ONLY JSON in exactly the shape you were given: " - '{"summary": str, "experience": [{"title", "employer", "bullets": [str]}], ' - '"projects": [{"name", "description", "tech": [str]}]}, same roles and projects in the same order.' + f"{_CV_TEXT_SHAPE}, same roles and projects in the same order." ), - user=(f"ISSUES TO FIX:\n{_issue_lines(findings)}\n\nCV TEXT (JSON):\n{json.dumps(text, ensure_ascii=False)}\n\n" + user=(f"ISSUES TO FIX:\n{_issue_lines(findings)}\n\nCV TEXT (JSON):\n{json.dumps(_cv_text(cv), ensure_ascii=False)}\n\n" f"TARGET JOB: {job.get('title', '')} at {job.get('company', '')}\n\n" f"CANDIDATE DATA (JSON):\n{json.dumps(candidate, ensure_ascii=False)}"), max_tokens=2048, label="revise_cv", ) + return _merge_cv_text(cv, data) + + +_CV_RUBRIC = ( + "1. RELEVANCE: the ad's three most important requirements are visibly answered in the summary and in the " + "first bullet of the most relevant role or project, in the ad's own words where the candidate data supports " + "them. 2. EVIDENCE: every bullet is a strong verb + what was built or done + a concrete result or scope that " + "the candidate data states; no generic claims ('collaborated with cross-functional teams', 'worked on " + "various projects'). 3. SUMMARY: a specific specialism and domain plus one concrete strength; no cliché " + "openers or closing filler, no pronouns. 4. PROJECTS: what it does, how it works, and a real result, framed " + "for this job's domain. 5. ORDER: the most job-relevant bullet comes first in each role." +) + +_LETTER_RUBRIC = ( + "1. SPECIFIC TO THEM: the 'why them' paragraph passes the swap test — if another employer's name could " + "replace theirs and it still reads fine, it fails. 2. EVIDENCE: two or three requirements named in the ad " + "are each proved by a named project or role from the candidate data, with a real result. 3. OPENING: a " + "specific hook tied to this employer, never 'I am writing to apply'. 4. ENDING: short and concrete; no perks " + "or salary. 5. LANGUAGE: active, confident, no hedging ('I believe', 'I feel'), no clichés or filler." +) + + +async def _critique_cv_text(cv: dict, candidate: dict, job: dict) -> tuple[dict | None, list[str]]: + """ + A critique-and-revise pass on writing quality: the model reviews the CV text against _CV_RUBRIC + and the ad, lists what falls short, and rewrites only those parts. Returns (revised CV or None, + issues found). The caller keeps the revision only if it passes the document check. + """ + data = await _llm_json( + system=( + "You are a demanding recruiter reviewing a tailored CV for one job. Judge the CV text against this " + f"rubric: {_CV_RUBRIC} " + "List each real shortfall in 'issues' (short, specific: where and what). Then rewrite ONLY the parts " + "with an issue in 'revised'; copy everything else exactly. STRICT: use only facts the candidate data " + "states — never add a number, technology, employer or project it doesn't, and never estimate one. Keep " + "names, the same roles and projects in the same order, and about the same length (the CV must still " + "fit one page). If nothing falls short, return an empty 'issues' list. Return ONLY JSON: " + f'{{"issues": [str], "revised": {_CV_TEXT_SHAPE}}}.' + ), + user=(f"TARGET JOB: {job.get('title', '')} at {job.get('company', '')}\n" + f"JOB DESCRIPTION:\n{job.get('description', '')[:2500]}\n\n" + f"CV TEXT (JSON):\n{json.dumps(_cv_text(cv), ensure_ascii=False)}\n\n" + f"CANDIDATE DATA (JSON):\n{json.dumps(candidate, ensure_ascii=False)}"), + max_tokens=3000, + label="critique_cv", + ) + if not isinstance(data, dict): + return None, [] + issues = [str(i).strip() for i in data.get("issues") or [] if str(i).strip()] + if not issues: + return None, [] + return _merge_cv_text(cv, data.get("revised")), issues + + +async def _critique_letter_text(letter: dict, persona: dict, job: dict, research: str) -> tuple[dict | None, list[str]]: + """The cover-letter counterpart of _critique_cv_text, judged against _LETTER_RUBRIC.""" + data = await _llm_json( + system=( + "You are a demanding recruiter reviewing a cover letter for one job. Judge it against this rubric: " + f"{_LETTER_RUBRIC} " + "List each real shortfall in 'issues' (short, specific: which paragraph and what). Then rewrite ONLY " + "the paragraphs with an issue; copy the others exactly. STRICT: use only facts from the candidate " + "data, the job ad and the company research — never add a number or technology none of them states. " + "Keep the same number of paragraphs and about the same length (one page). If nothing falls short, " + 'return an empty \'issues\' list. Return ONLY JSON: {"issues": [str], "paragraphs": [str]}.' + ), + user=(f"JOB: {job.get('title', '')} at {job.get('company', '')}\n" + f"DESCRIPTION: {job.get('description', '')[:2500]}\n\n" + f"LETTER (JSON):\n{json.dumps({'paragraphs': letter.get('paragraphs', [])}, ensure_ascii=False)}\n\n" + f"CANDIDATE PERSONA (JSON): {json.dumps(_llm_persona(persona), ensure_ascii=False)}" + + (f"\n\nCOMPANY RESEARCH:\n{research}" if research else "")), + max_tokens=1500, + label="critique_letter", + ) + if not isinstance(data, dict): + return None, [] + issues = [str(i).strip() for i in data.get("issues") or [] if str(i).strip()] + paragraphs = data.get("paragraphs") + if not issues: + return None, [] + if (not isinstance(paragraphs, list) or len(paragraphs) != len(letter.get("paragraphs") or []) + or not all(isinstance(p, str) and p.strip() for p in paragraphs)): + return None, issues + return {**letter, "paragraphs": [p.strip() for p in paragraphs]}, issues + + +def _merge_cv_text(cv: dict, data) -> dict | None: + """The CV with a revision's summary, bullets and project text, or None when the revision doesn't + keep the CV's shape (same roles and projects, in the same order).""" if not isinstance(data, dict): return None + text = _cv_text(cv) roles, projects = data.get("experience"), data.get("projects") if (not isinstance(roles, list) or len(roles) != len(text["experience"]) or not isinstance(projects, list) or len(projects) != len(text["projects"])): @@ -684,6 +785,25 @@ async def _revise_letter_text(letter: dict, findings: list[dict], persona: dict, return {**letter, "paragraphs": [p.strip() for p in paragraphs]} +# Which documents get the critique-and-revise pass on writing quality: DOC_CRITIQUE is a comma-separated +# list of "cv" and "letter" ("0" or empty for none). Letters only by default. On the quality eval +# (evals/test_doc_quality_eval.py, three runs of six adverts) the critiqued letter beat its draft every run +# (4-2, 5-1, then 2-1 with 3 ties when judged in both orders); CVs were mixed (1-3, 4-0, 2-0), so for CVs +# it isn't yet worth the extra model call. +_DOC_CRITIQUE = {k.strip() for k in os.environ.get("DOC_CRITIQUE", "letter").split(",")} & {"cv", "letter"} +_CRITIQUE_GROWTH = 1.15 # a rewrite may run this much longer before it would crowd the page + + +def _critique_holds(doc_type: str, before: dict, after: dict, persona: dict, job: dict, research: str = "") -> bool: + """A quality rewrite is kept only if the document check finds nothing new in it and it hasn't grown + past _CRITIQUE_GROWTH.""" + def found(doc): + return doc_check.check_facts(doc_type, doc, persona, job, research) + doc_check.check_style(doc_type, doc) + text = (lambda d: d.get("paragraphs")) if doc_type == "cover_letter" else _cv_text + return (_finding_keys(found(after)) <= _finding_keys(found(before)) + and _word_count(text(after)) <= _CRITIQUE_GROWTH * max(1, _word_count(text(before)))) + + def _better(before: list[dict], after: list[dict]) -> bool: """A revision is kept only when it clears findings and introduces none of its own.""" return len(after) < len(before) and _finding_keys(after) <= _finding_keys(before) @@ -1155,6 +1275,16 @@ async def execute_tool(name: str, tool_input: dict) -> dict: ) letter["signoff"] = "Yours faithfully," if _generic_salutation else "Yours sincerely," + # Critique and revise against the rubric; kept only if it adds no unsupported claim and + # stays about the same length. + review: dict = {"issues": [], "kept": False} + if data and "letter" in _DOC_CRITIQUE: + revised, review["issues"] = await _critique_letter_text(letter, persona, job, company_research) + if revised is not None: + review["kept"] = _critique_holds("cover_letter", letter, revised, persona, job, company_research) + if review["kept"]: + letter = revised + # Numbers and tools the persona, the ad and the company research don't state, and # machine-sounding words: one targeted revision, kept only if it clears them. revision = "fallback" @@ -1175,9 +1305,10 @@ async def execute_tool(name: str, tool_input: dict) -> dict: letter = revised (logger.warning if data is None else logger.info)( - "cover_letter: llm_used=%s, paragraphs=%d, archetype=%s, level=%s, revision=%s | job=%r", - data is not None, len(letter["paragraphs"]), archetype, level, revision, - (job.get("title") or "")[:60], + "cover_letter: llm_used=%s, paragraphs=%d, archetype=%s, level=%s, critique=%d issues %s, revision=%s " + "| job=%r", + data is not None, len(letter["paragraphs"]), archetype, level, len(review["issues"]), + "kept" if review["kept"] else "not kept", revision, (job.get("title") or "")[:60], ) return { @@ -1187,6 +1318,7 @@ async def execute_tool(name: str, tool_input: dict) -> dict: "candidate_name": full_name, "letter": letter, "archetype": archetype, + "review": review, } elif name == "generate_skill_roadmap": @@ -1375,6 +1507,15 @@ async def execute_tool(name: str, tool_input: dict) -> dict: added_skills: list[str] = [] if chosen: cv["skills"], added_skills = _top_up_skills(cv["skills"], base_cv["skills"], cv, job) + # Critique and revise against the rubric; kept only if it adds no unsupported claim and + # stays about the same length (the page fit is measured on the final text). + review: dict = {"issues": [], "kept": False} + if not used_fallback and "cv" in _DOC_CRITIQUE: + revised, review["issues"] = await _critique_cv_text(cv, llm_cv, job) + if revised is not None: + review["kept"] = _critique_holds("cv", cv, revised, persona, job) + if review["kept"]: + cv = revised # Check the text against the candidate's data; one targeted revision when it flags something. revision = "fallback" if not used_fallback: @@ -1396,12 +1537,13 @@ async def execute_tool(name: str, tool_input: dict) -> dict: # same projects recur across roles (the model is meant to tailor the selection). final_proj_names = [(p.get("name") or "")[:40] for p in (cv.get("projects") or [])][:5] (logger.warning if used_fallback else logger.info)( - "tailored_cv: fallback=%s, level=%s, skills chosen=%s, added=%s, revision=%s, projects base=%d model=%d " - "final=%d %s | job=%r", - used_fallback, level, chosen is not None, added_skills, revision, base_proj_n, model_proj_n, + "tailored_cv: fallback=%s, level=%s, skills chosen=%s, added=%s, critique=%d issues %s, revision=%s, " + "projects base=%d model=%d final=%d %s | job=%r", + used_fallback, level, chosen is not None, added_skills, len(review["issues"]), + "kept" if review["kept"] else "not kept", revision, base_proj_n, model_proj_n, len(cv.get("projects") or []), final_proj_names, (job.get("title") or "")[:60], ) - return {"cv": cv, "template_id": template_id} + return {"cv": cv, "template_id": template_id, "review": review} return {"error": f"Unknown tool: {name}"} diff --git a/api/evals/test_doc_quality_eval.py b/api/evals/test_doc_quality_eval.py new file mode 100644 index 0000000..1af1b36 --- /dev/null +++ b/api/evals/test_doc_quality_eval.py @@ -0,0 +1,154 @@ +""" +Does the critique-and-revise pass make tailored CVs and cover letters better? + +For each advert below, the graduate profile from tests/test_documents.py is tailored with the real +model and the critique off (the draft). The critique then runs on the draft as it does in +production, including the guard that drops a rewrite with an unsupported claim or that runs long. +Where it changed something, a judge sees the draft and the revision blind, in both orders (a version +wins only when both agree) and says which better serves the advert. Every run writes evals/last_doc_run.json. + +The bar: no kept revision may carry a claim the document check flags that the draft didn't, and for +cover letters (which get the critique by default) the revision must win more often than it loses. +CVs are measured only: across the first three runs they were mixed (draft 3-1, then revision 4-0 and +2-0), so the CV critique is off by default. +""" +import asyncio +import json +import pathlib + +import pytest + +from app import agent, doc_check +from test_documents import GRADUATE + +HERE = pathlib.Path(__file__).parent + +ADS = [ + {"title": "Graduate Backend Engineer", "company": "Harbourline", "location": "London", + "description": "Harbourline builds the booking platform behind 400 ferry routes. As a graduate backend engineer " + "you will design and ship Python services (FastAPI) backed by PostgreSQL, write tests with " + "pytest, and run them in Docker on AWS. You will own features end to end, from API design to " + "monitoring in production. We look for: a degree in a STEM subject or equivalent experience; " + "solid Python; SQL; an understanding of REST API design; curiosity about distributed systems."}, + {"title": "Junior Data Engineer", "company": "Meridian Health Analytics", "location": "Bristol", + "description": "Join a small team turning hospital operations data into dashboards that clinicians use daily. " + "You will build ETL pipelines in Python and SQL, model data in PostgreSQL, schedule jobs with " + "Airflow and keep data quality high with automated tests. Nice to have: pandas, dbt, AWS. " + "You care about getting numbers right and explaining them to non-technical colleagues."}, + {"title": "Graduate Machine Learning Engineer", "company": "Signalbox Rail", "location": "Manchester", + "description": "Signalbox predicts disruption on UK rail networks. You will train and evaluate forecasting " + "models in Python with PyTorch and scikit-learn, build data pipelines for live feeds, and help " + "deploy models behind APIs. We value rigorous evaluation, clear write-ups and pragmatic " + "engineering. Requirements: Python, machine learning fundamentals, SQL, Git."}, + {"title": "Graduate Frontend Developer", "company": "Larder", "location": "Remote, UK", + "description": "Larder helps households waste less food. You will build features in React and Next.js with " + "TypeScript, write component and end-to-end tests (Jest, Playwright), and work closely with " + "design. You will ship to thousands of users weekly. We want: TypeScript, React, attention to " + "accessibility and performance, and a portfolio of things you have built."}, + {"title": "Graduate Software Engineer, Trading Systems", "company": "Quayside Capital", "location": "London", + "description": "Write low-latency, well-tested software for our trading platform in Java and Python. You will " + "work on market data services, use data structures and algorithms to make them fast, and pair " + "with traders to understand requirements. Strong programming fundamentals, a quantitative " + "mindset and a 2:1 or above in Computer Science, Maths or Physics."}, + {"title": "Graduate DevOps Engineer", "company": "Cloudrail", "location": "Leeds", + "description": "Help our platform team automate everything. You will build CI/CD pipelines with GitHub " + "Actions, containerise services with Docker, manage AWS infrastructure and write Python " + "tooling. Kubernetes and Terraform experience is a bonus. You enjoy making other engineers " + "faster and systems more reliable."}, +] + +JUDGE = ( + "You are an experienced UK graduate recruiter. You see one job advert and two versions (A and B) of the same " + "candidate's {kind}, written from the same facts. Decide which version better serves THIS advert: it answers " + "the advert's main requirements visibly and in the advert's terms, proves claims with concrete specifics and " + "results, reads as written by a person (no clichés, filler or hedging), and is concise. Ignore formatting. " + 'If they are equally good, say tie. Return ONLY JSON: {{"better": "A" | "B" | "tie", "reason": str}}.' +) + + +def _candidate(job): + base = agent._persona_to_cvdoc(GRADUATE) + return {**base, "contact": {}, "projects": agent._rank_projects_by_relevance(job, agent._persona_projects_for_llm(GRADUATE))} + + +async def _judge_once(kind: str, job: dict, draft, revised, revised_first: bool) -> tuple[str, str]: + a, b = (revised, draft) if revised_first else (draft, revised) + verdict = await agent._llm_json( + system=JUDGE.format(kind=kind), + user=f"ADVERT: {job['title']} at {job['company']}\n{job['description']}\n\n" + f"VERSION A:\n{json.dumps(a, ensure_ascii=False)}\n\nVERSION B:\n{json.dumps(b, ensure_ascii=False)}", + max_tokens=600, label="eval_judge") + pick, reason = (verdict or {}).get("better", "tie"), str((verdict or {}).get("reason", "")) + if pick == "tie" or pick not in ("A", "B"): + return "tie", reason + return ("revised" if (pick == "A") == revised_first else "draft"), reason + + +async def _judge(kind: str, job: dict, draft, revised) -> tuple[str, list[str]]: + """Judged in both orders; a version wins only when both say so (position bias cancels out).""" + (first, r1), (second, r2) = await asyncio.gather(_judge_once(kind, job, draft, revised, True), + _judge_once(kind, job, draft, revised, False)) + return (first if first == second else "tie"), [r1, r2] + + +async def _one(i: int, job: dict) -> dict: + out = {"job": f"{job['title']} @ {job['company']}"} + cv = (await agent.execute_tool("generate_tailored_cv", {"job": job, "persona": GRADUATE}))["cv"] + letter = (await agent.execute_tool("generate_cover_letter", {"job": job, "persona": GRADUATE}))["letter"] + + revised_cv, cv_issues = await agent._critique_cv_text(cv, _candidate(job), job) + cv_kept = revised_cv is not None and agent._critique_holds("cv", cv, revised_cv, GRADUATE, job) + revised_letter, letter_issues = await agent._critique_letter_text(letter, GRADUATE, job, "") + letter_kept = revised_letter is not None and agent._critique_holds("cover_letter", letter, revised_letter, GRADUATE, job) + + for kind, draft, revised, issues, kept, text in ( + ("CV", cv, revised_cv, cv_issues, cv_kept, agent._cv_text), + ("cover letter", letter, revised_letter, letter_issues, letter_kept, lambda d: d.get("paragraphs")), + ): + key = "cv" if kind == "CV" else "letter" + new_flags = [] + if revised is not None: + before = doc_check.check_facts("cv" if key == "cv" else "cover_letter", draft, GRADUATE, job) + after = doc_check.check_facts("cv" if key == "cv" else "cover_letter", revised, GRADUATE, job) + new_flags = sorted(agent._finding_keys(after) - agent._finding_keys(before)) + result = {"issues": issues, "rewrote": revised is not None, "kept": kept, "new_flags": [list(f) for f in new_flags], + "draft": text(draft), "revised": text(revised) if revised is not None else None} + if kept: + result["verdict"], result["reasons"] = await _judge(kind, job, text(draft), text(revised)) + out[key] = result + return out + + +@pytest.fixture +def no_research(monkeypatch): + async def none(*args, **kwargs): + return "" + monkeypatch.setattr(agent, "_research_company", none) + monkeypatch.setattr(agent, "_DOC_CRITIQUE", set()) # drafts without the critique; it runs separately + + +def test_critique_improves_documents(no_research): + async def run(): + sem = asyncio.Semaphore(3) + + async def guarded(i, job): + async with sem: + return await _one(i, job) + return await asyncio.gather(*(guarded(i, j) for i, j in enumerate(ADS))) + + results = asyncio.run(run()) + (HERE / "last_doc_run.json").write_text(json.dumps(results, ensure_ascii=False, indent=1), encoding="utf-8") + + score = {} + for key in ("cv", "letter"): + verdicts = [r[key]["verdict"] for r in results if "verdict" in r[key]] + score[key] = (verdicts.count("revised"), verdicts.count("draft")) + print(f"{key}: issues found {sum(bool(r[key]['issues']) for r in results)}/{len(results)}, " + f"rewrites kept {sum(r[key]['kept'] for r in results)}, revised won {score[key][0]}, " + f"lost {score[key][1]}, tied {verdicts.count('tie')}") + # The guard must have dropped any rewrite with a new unsupported claim. + assert not any(r[key]["kept"] and r[key]["new_flags"] for r in results) + # Letters get the critique by default (agent._DOC_CRITIQUE), so they must gain from it; CVs are measured + # so a change to the CV critique can be judged before it's turned on. + wins, losses = score["letter"] + assert wins > losses, f"letters: the critique didn't beat its drafts ({wins} wins, {losses} losses)" diff --git a/api/tests/conftest.py b/api/tests/conftest.py index a2a8069..dad5995 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -71,6 +71,8 @@ def do_POST(self): "GROQ_API_KEY": "test-key", "OLLAMA_HOST": f"http://127.0.0.1:{_ollama.server_address[1]}", "ANONYMIZED_TELEMETRY": "false", + # The critique pass is one more model call per document; tests that cover it turn it on. + "DOC_CRITIQUE": "0", }) diff --git a/api/tests/test_doc_check.py b/api/tests/test_doc_check.py index 5fedb1a..fa216b1 100644 --- a/api/tests/test_doc_check.py +++ b/api/tests/test_doc_check.py @@ -223,3 +223,66 @@ def test_a_letter_with_a_flagged_word_is_revised(replies): json.dumps({"paragraphs": ["I use FastAPI daily.", "Your team of 40 engineers ships daily."]})] letter = asyncio.run(agent.execute_tool("generate_cover_letter", {"job": AD, "persona": GRADUATE}))["letter"] assert letter["paragraphs"] == ["I use FastAPI daily.", "Your team of 40 engineers ships daily."] + + +# ── Critique and revise (writing quality) ───────────────────────────────────────────────────── + +@pytest.fixture +def critique(monkeypatch, replies): + monkeypatch.setattr(agent, "_DOC_CRITIQUE", {"cv", "letter"}) + return replies + + +GOOD_BULLET = "Built a FastAPI service ingesting 40k readings a day into PostgreSQL." + + +def _critique_reply(summary: str, bullet: str = GOOD_BULLET, issues=("Summary misses the ad's FastAPI focus",)) -> str: + text = json.loads(_text_reply(bullet)) + text["summary"] = summary + return json.dumps({"issues": list(issues), "revised": text}) + + +def _generate(kind: str): + tool = "generate_tailored_cv" if kind == "cv" else "generate_cover_letter" + return asyncio.run(agent.execute_tool(tool, {"job": AD, "persona": GRADUATE})) + + +def test_a_critique_rewrite_that_holds_up_is_kept(critique): + better = "Computer Science graduate building Python and FastAPI services on PostgreSQL." + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), _critique_reply(better)] + out = _generate("cv") + assert out["cv"]["summary"] == better + assert out["review"] == {"issues": ["Summary misses the ad's FastAPI focus"], "kept": True} + assert "demanding recruiter" in critique.prompts[1] and "RELEVANCE" in critique.prompts[1] + + +def test_a_critique_rewrite_that_invents_a_claim_is_dropped(critique): + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), + _critique_reply("Graduate who cut latency by 35% with FastAPI.")] + out = _generate("cv") + assert out["cv"]["summary"] == "Computer Science graduate who builds Python and FastAPI services." + assert out["review"]["kept"] is False + + +def test_a_critique_rewrite_that_runs_long_is_dropped(critique): + long_summary = "Computer Science graduate who builds Python and FastAPI services. " * 6 + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), _critique_reply(long_summary)] + assert _generate("cv")["review"]["kept"] is False + + +def test_no_issues_leaves_the_cv_alone(critique): + critique.queue = [json.dumps(_model_cv(GOOD_BULLET)), json.dumps({"issues": [], "revised": {}})] + out = _generate("cv") + assert out["cv"]["experience"][0]["bullets"] == [GOOD_BULLET] + assert out["review"] == {"issues": [], "kept": False} + + +def test_the_letter_critique_rewrites_the_flagged_paragraph(critique): + first = {"salutation": "Dear Hiring Manager,", "signoff": "Yours faithfully,", "headline": "Graduate", + "paragraphs": ["I am writing to apply for this role.", "I build FastAPI services on PostgreSQL."]} + critique.queue = [json.dumps(first), json.dumps({ + "issues": ["Opening is generic"], + "paragraphs": ["Northwind's API platform is where I want to build.", "I build FastAPI services on PostgreSQL."]})] + out = _generate("letter") + assert out["letter"]["paragraphs"][0] == "Northwind's API platform is where I want to build." + assert out["review"] == {"issues": ["Opening is generic"], "kept": True} diff --git a/frontend/e2e/smoke.spec.ts b/frontend/e2e/smoke.spec.ts index 482f0cc..122abda 100644 --- a/frontend/e2e/smoke.spec.ts +++ b/frontend/e2e/smoke.spec.ts @@ -233,7 +233,8 @@ test("the job workspace tailors a CV and checks it against the persona", async ( "GET /api/jobs/1/sponsorship": { sponsorship: "offered", evidence: "", clearance: "", licensed_sponsor: false, sponsor_name: "", mark: "" }, "POST /api/jobs/analyze": { sse: [{ type: "job_fit", ranked_jobs: [{ fit_score: 84, level_match: "strong", matched_skills: ["Python"], skill_gaps: ["Kubernetes"], reasoning: "Strong backend overlap." }] }] }, - "POST /api/jobs/tailored-cv": { sse: [{ type: "tailored_cv", cv }] }, + "POST /api/jobs/tailored-cv": { sse: [{ type: "tailored_cv", cv, + review: { issues: ["Summary didn't name FastAPI"], kept: true } }] }, "POST /api/documents/preview": { html: "CV" }, "POST /api/documents/check": (request) => { checked.push(request.postDataJSON()); @@ -252,7 +253,9 @@ test("the job workspace tailors a CV and checks it against the persona", async ( const checks = page.getByRole("button", { name: /^Checks/ }); await expect(checks).toContainText("1 thing to check"); await expect(checks).toContainText("Keywords 2 of 3 you have"); + await expect(checks).toContainText("Reviewed for this ad: 1 fix"); await checks.click(); + await expect(page.getByText("Summary didn't name FastAPI")).toBeVisible(); await expect(page.getByText("“35%” isn't in your persona")).toBeVisible(); await expect(page.getByText("Docker", { exact: true })).toBeVisible(); expect(checked[0]).toMatchObject({ type: "cv", doc: { name: "Sam Okafor" }, job: { title: job.title } }); diff --git a/frontend/src/app/jobs/[id]/page.tsx b/frontend/src/app/jobs/[id]/page.tsx index 3580db9..55c67e6 100644 --- a/frontend/src/app/jobs/[id]/page.tsx +++ b/frontend/src/app/jobs/[id]/page.tsx @@ -51,6 +51,12 @@ async function downloadPdf(docId: string, filename: string) { URL.revokeObjectURL(url); } +/** The issues a kept critique rewrite fixed, from the generation event's `review`. */ +function keptIssues(review: unknown): string[] { + const r = review as { issues?: unknown; kept?: unknown } | undefined; + return r?.kept === true && Array.isArray(r.issues) ? r.issues.filter((i): i is string => typeof i === "string") : []; +} + const LEVEL_MATCH: Record = { strong: "strong", ok: "reasonable", reach: "a stretch" }; const EXPOSURE_NOTE = { @@ -82,6 +88,8 @@ export default function JobDetailPage() { const [cvError, setCvError] = useState(null); const [letterError, setLetterError] = useState(null); const [cvChoice, setCvChoice] = useState("tailored"); + // What the critique-and-revise pass found and fixed, when its rewrite was kept. + const [reviewed, setReviewed] = useState<{ cv: string[]; letter: string[] }>({ cv: [], letter: [] }); // PDF state const [cvDocId, setCvDocId] = useState(null); @@ -179,7 +187,11 @@ export default function JobDetailPage() { let got = false; try { await postStream("/api/jobs/tailored-cv", { job }, (ev) => { - if (ev.type === "tailored_cv" && ev.cv) { setCvDoc(ev.cv as CVDoc); got = true; } + if (ev.type === "tailored_cv" && ev.cv) { + setCvDoc(ev.cv as CVDoc); + setReviewed((r) => ({ ...r, cv: keptIssues(ev.review) })); + got = true; + } }); if (!got) setCvError("The tailored CV didn't come through."); } catch (e) { @@ -197,7 +209,11 @@ export default function JobDetailPage() { let got = false; try { await postStream("/api/jobs/cover-letter", { job }, (ev) => { - if (ev.type === "cover_letter" && ev.letter) { setLetterDoc(ev.letter as LetterDoc); got = true; } + if (ev.type === "cover_letter" && ev.letter) { + setLetterDoc(ev.letter as LetterDoc); + setReviewed((r) => ({ ...r, letter: keptIssues(ev.review) })); + got = true; + } }); if (!got) setLetterError("The cover letter didn't come through."); } catch (e) { @@ -534,7 +550,7 @@ export default function JobDetailPage() { {docToolbar("cv")} - {cvDoc && !streamingCv && } + {cvDoc && !streamingCv && } {streamingCv ? docSkeleton("Tailoring your CV for this role…") : cvDoc ? ( { setCvDoc(c); setCvDocId(null); }} /> ) : ( @@ -562,7 +578,7 @@ export default function JobDetailPage() { {docToolbar("cover_letter")} - {letterDoc && !streamingLetter && } + {letterDoc && !streamingLetter && } {streamingLetter ? docSkeleton("Writing your cover letter…") : letterDoc ? ( { setLetterDoc(l); setLetterDocId(null); }} /> diff --git a/frontend/src/components/DocChecks.tsx b/frontend/src/components/DocChecks.tsx index f59ee1d..1dbe716 100644 --- a/frontend/src/components/DocChecks.tsx +++ b/frontend/src/components/DocChecks.tsx @@ -30,8 +30,10 @@ function Pill({ tone, children }: { tone: "ok" | "warn" | "neutral"; children: R * claims your persona doesn't back, machine-sounding words, and the PDF read back as an applicant * tracking system reads it. Re-runs a moment after each edit. */ -export default function DocChecks({ kind, doc, job, templateId = "classic" }: { +export default function DocChecks({ kind, doc, job, templateId = "classic", reviewed = [] }: { kind: "cv" | "cover_letter"; doc: object; job: Job; templateId?: string; + /** What the review against the ad found and rewrote while the document was written. */ + reviewed?: string[]; }) { const [report, setReport] = useState(null); const [checking, setChecking] = useState(false); @@ -84,6 +86,9 @@ export default function DocChecks({ kind, doc, job, templateId = "classic" }: {