From a98a103f718affcdba719e529e5ac15d06e5ae28 Mon Sep 17 00:00:00 2001 From: Timothy Oluwatobi Ojebiyi Date: Thu, 15 Oct 2026 17:22:14 +0100 Subject: [PATCH 1/3] test: CV and cover-letter rendering, from profile to PDF A graduate profile shaped like a real one goes through the tailoring tool, the HTML template and WeasyPrint: section order, links, project prose clean-up, pronoun-free summary, escaping, length limits, the guardrails on what the model may change, cover-letter sign-off and paragraphs, page counts in the template font, the fpdf2 fallback and the PDF endpoints. CI installs Pango and Liberation fonts so the PDF tests run there with production's font. Two known gaps are strict xfails: skill groups after the sixth are dropped, and a full graduate CV spills onto a second page. --- .github/workflows/ci.yml | 4 + api/tests/test_documents.py | 385 +++++++++++++++++++++++++++++++++++- 2 files changed, 387 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70a7e4f..fa8c3fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,10 @@ jobs: python-version: "3.13" cache: pip cache-dependency-path: api/requirements*.txt + # WeasyPrint's libraries and the CV template's font, as in api/Dockerfile: the PDF tests + # measure page counts, so they must lay out with the same font as production. + - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends + libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 fonts-liberation - run: pip install -r requirements.txt -r requirements-dev.txt - run: pytest diff --git a/api/tests/test_documents.py b/api/tests/test_documents.py index 80dbb61..1cdf257 100644 --- a/api/tests/test_documents.py +++ b/api/tests/test_documents.py @@ -1,6 +1,134 @@ -"""CV document building: persona → CVDoc → rendered HTML.""" -from app import agent, doc_render +"""CV and cover-letter documents: profile → CVDoc / LetterDoc → HTML → PDF.""" +import asyncio +import copy +import json +import os +import re +import fitz # pymupdf +import pytest + +from app import agent, doc_render, pdf_generator + +# A graduate profile shaped like a real one: portfolio-style project prose (headers,
, first +# person, a "Technologies" section), one internship, and more skill groups than the CV has room for. +GRADUATE = { + "core_info": {"first_name": "Sam", "last_name": "Okafor", "email": "sam@example.com", + "phone": "+44 7700 900456", "city": "Bristol", "linkedin": "linkedin.com/in/sam-okafor", + "github": "github.com/samokafor", "website": "samokafor.dev"}, + "summary": "I am a Computer Science graduate who builds full-stack web apps and data pipelines, " + "seeking a role where I can grow my skills.", + "education": [{"institution": "Northfield University", "degree": "BSc (Hons) Computer Science", + "start_date": "09/2022", "end_date": "06/2025", "grade": "2:1"}], + "history": [{"employer": "Brightwave Analytics", "title": "Software Engineering Intern", + "start_date": "06/2024", "end_date": "09/2024", + "achievements": ["Built a FastAPI service that ingests 40k sensor readings a day into PostgreSQL", + "Cut dashboard load time from 6 s to 1.5 s by adding Redis caching", + "Wrote integration tests that raised coverage from 52% to 81%"]}], + "projects": [ + {"name": "Pantry Planner", "url": "https://github.com/samokafor/pantry-planner", + "problem_solved": "What problem did you solve?
I built a meal planner that turns the food already in a " + "kitchen into a week of recipes, so less of it is thrown away. It matches pantry items " + "against 12,000 recipes with TF-IDF similarity and ranks them by what expires first." + "
How it works
1) Scan a receipt 2) Match recipes", + "outcomes": "Outcomes: Used by 300 people a week during a university trial.
Technologies Frontend: " + "Next.js, React Backend: FastAPI, PostgreSQL ML: scikit-learn"}, + {"name": "Rail Delay Forecaster", "url": "rail-delays.example.app", "technologies": ["Python", "PyTorch"], + "problem_solved": "Forecasts arrival delays for 40 stations from live rail feeds. A gradient-boosted " + "model retrains nightly on the previous month of running data.", + "outcomes": "Mean absolute error of 2.1 minutes on held-out weeks."}, + {"name": "Campus Lost & Found", + "problem_solved": "A web app where students report and claim lost items, with image matching to suggest " + "likely owners. Built with Django and PostgreSQL in Docker.", + "outcomes": "450 items returned in the first term."}, + {"name": "Chess Engine", + "problem_solved": "I wrote a chess engine in Java using alpha-beta search with move ordering and a " + "transposition table. It plays at about 1900 on Lichess."}, + ], + "skills": [ + {"category": "Languages", "skills": ["Python", "TypeScript", "Java", "SQL"]}, + {"category": "Frontend", "skills": ["React", "Next.js", "Tailwind CSS"]}, + {"category": "Backend", "skills": ["FastAPI", "Node.js", "Spring Boot"]}, + {"category": "Data", "skills": ["PostgreSQL", "Redis", "pandas"]}, + {"category": "Machine Learning", "skills": ["scikit-learn", "PyTorch"]}, + {"category": "Testing", "skills": ["pytest", "Jest", "Playwright"]}, + {"category": "Cloud", "skills": ["AWS", "Docker"]}, + {"category": "Tools", "skills": ["Git", "GitHub Actions", "CI/CD"]}, + ], + "certifications": [{"name": "AWS Cloud Practitioner", "issuing_organization": "Amazon Web Services", + "issue_date": "2025-03"}], +} +JOB = {"job_id": "1", "title": "Graduate Software Engineer", "company": "Acme Robotics", "location": "Cambridge, UK", + "description": "Python, FastAPI and PostgreSQL services for our robot fleet; machine learning a plus."} + +_WORDS = ("designed shipped measured reduced automated migrated profiled documented tested deployed " + "monitored refactored").split() + + +def _sentence(n_words: int, seed: int) -> str: + return " ".join(_WORDS[(seed + i) % len(_WORDS)] for i in range(n_words)).capitalize() + + +def _letter_paragraphs(words_each=(60, 80, 140, 40)) -> list[str]: + return [_sentence(n, i) + "." for i, n in enumerate(words_each)] + + +def _full_cvdoc(seniority: str) -> dict: + """A CVDoc at every length limit: what a verbose model can hand the renderer.""" + return { + "name": "Sam Okafor", "seniority": seniority, + "contact": {"email": "sam@example.com", "phone": "+44 7700 900456", "location": "Bristol", + "linkedin": "linkedin.com/in/sam-okafor", "github": "github.com/samokafor"}, + "summary": " ".join(_sentence(25, i) + "." for i in range(3)), + "skills": [{"category": f"Group {g}", "items": [f"Skill{g}{i}" for i in range(14)]} for g in range(6)], + "experience": [{"title": f"Engineer {r}", "employer": f"Employer {r}", "location": "London", + "start_date": f"01/{2015 + 2 * r}", "end_date": f"12/{2016 + 2 * r}", + "bullets": [_sentence(24, r + b) for b in range(4)]} for r in range(4)], + "projects": [{"name": f"Project {p}", "description": _sentence(80, p) + ".", + "tech": ["Python", "FastAPI", "PostgreSQL", "Docker", "React", "Redis", "AWS", "pytest"], + "link": f"github.com/samokafor/p{p}"} for p in range(3)], + "education": [{"degree": "BSc (Hons) Computer Science", "institution": "Northfield University", + "year": "06/2014", "grade": "First"}], + "certifications": [{"name": f"Certificate {c}", "issuer": "Issuer", "year": "2020"} for c in range(3)], + } + + +@pytest.fixture +def llm(monkeypatch): + """The model's reply: set llm.reply to the raw text it returns. Prompts are kept in llm.prompts.""" + class Fake: + reply = "{}" + prompts: list[str] = [] + + async def fake_llm(system, user, max_tokens=2048): + Fake.prompts.append(system + "\n" + user) + return Fake.reply + + async def no_research(*args, **kwargs): + return "" + monkeypatch.setattr(agent, "_llm", fake_llm) + monkeypatch.setattr(agent, "_research_company", no_research) + return Fake + + +def _tailored_cv(persona=GRADUATE) -> dict: + return asyncio.run(agent.execute_tool("generate_tailored_cv", {"job": JOB, "persona": persona}))["cv"] + + +def _cover_letter(persona=GRADUATE) -> dict: + return asyncio.run(agent.execute_tool("generate_cover_letter", {"job": JOB, "persona": persona}))["letter"] + + +def _headings(html: str) -> list[str]: + return re.findall(r"

(.*?)

", html) + + +def _letter_body(html: str) -> list[str]: + body = re.search(r'
(.*?)
', html, re.S).group(1) + return re.findall(r"

(.*?)

", body, re.S) + + +# ── CV: profile → document ─────────────────────────────────────────────────────── def test_certifications_keep_issuer_and_date_through_to_the_rendered_cv(): persona = {"core_info": {"first_name": "Ada", "last_name": "Lovelace"}, @@ -14,3 +142,256 @@ def test_certifications_keep_issuer_and_date_through_to_the_rendered_cv(): ] html = doc_render.render_cv_html(cv) assert "AWS Solutions Architect" in html and "Amazon" in html and "05/2024" in html + + +def test_a_graduate_cv_leads_with_education_and_links_every_profile(llm): + llm.reply = "not json" # the model failed: the CV is built straight from the profile + html = doc_render.render_cv_html(_tailored_cv()) + + assert _headings(html) == ["Professional Summary", "Education", "Work Experience", "Projects", "Skills", + "Certifications"] + assert '

Sam Okafor

' in html + for href in ("mailto:sam@example.com", "tel:+447700900456", "https://linkedin.com/in/sam-okafor", + "https://github.com/samokafor", "https://samokafor.dev"): + assert f'href="{href}"' in html + assert "BSc (Hons) Computer Science | 06/2025 | Northfield University" in html and "Grade: 2:1" in html + assert "Software Engineering Intern | Brightwave Analytics | 06/2024 – 09/2024" in html + assert "Cut dashboard load time from 6 s to 1.5 s" in html + assert "AWS Cloud Practitioner | Amazon Web Services | 03/2025" in html + + +def test_the_summary_loses_its_pronouns(llm): + llm.reply = "{}" + summary = re.search(r'

(.*?)

', doc_render.render_cv_html(_tailored_cv())).group(1) + assert summary == "Computer Science graduate who builds full-stack web apps and data pipelines." + + +def test_portfolio_prose_becomes_two_clean_sentences_with_labelled_links(llm): + llm.reply = "{}" + html = doc_render.render_cv_html(_tailored_cv()) + projects = html.split("

Projects

")[1].split("")[0] + + assert re.findall(r'
(.*?)
', projects) == [ + "Pantry Planner", "Rail Delay Forecaster", "Campus Lost & Found"] # three at most, escaped + assert ("Built a meal planner that turns the food already in a kitchen into a week of recipes, so less of it " + "is thrown away. It matches pantry items against 12,000 recipes with TF-IDF similarity and ranks them " + "by what expires first.") in projects + assert "What problem" not in projects and "How it works" not in projects and "<br" not in projects + assert 'GitHub' in projects + assert 'Live Demo' in projects + + +def test_the_model_tailors_the_cv_but_cannot_touch_contact_details_skills_or_links(llm): + llm.reply = "Here is the CV:\n```json\n" + json.dumps({ + "name": "Sam Okafor", "headline": "", + "contact": {"email": "invented@example.org", "phone": "+1 555 0100"}, + "summary": "Computer Science graduate specialising in Python back-end services.", + "skills": [{"category": "Languages", "items": ["Python"]}], + "experience": [{"title": "Software Engineering Intern", "employer": "Brightwave Analytics", + "start_date": "06/2024", "end_date": "09/2024", + "bullets": ["Built a FastAPI service ingesting 40k sensor readings a day into PostgreSQL"]}], + "education": GRADUATE["education"], + "projects": [{"name": "Rail Delay Forecaster", "description": "Forecasts delays.", "tech": [], + "link": "https://elsewhere.example"}, + {"name": "Pantry Planner (meal planning)", "description": "Plans meals.", "tech": ["Next.js"], + "link": ""}], + }) + "\n```" + cv = _tailored_cv() + base = agent._persona_to_cvdoc(GRADUATE) + + assert cv["summary"] == "Computer Science graduate specialising in Python back-end services." # tailored + assert cv["contact"] == base["contact"] + assert cv["skills"] == base["skills"] # the model dropped seven groups + assert [(p["name"], p["link"], p["tech"]) for p in cv["projects"]] == [ + ("Rail Delay Forecaster", "rail-delays.example.app", ["Python", "PyTorch"]), + ("Pantry Planner (meal planning)", "https://github.com/samokafor/pantry-planner", ["Next.js"]), + ] + assert cv["seniority"] == "junior" + assert "sam@example.com" not in llm.prompts[-1] and "7700 900456" not in llm.prompts[-1] + + +@pytest.mark.xfail(strict=True, reason="skill groups after the sixth are dropped (doc_render._MAX_SKILL_GROUPS)") +def test_every_skill_in_the_profile_reaches_the_cv(llm): + html = doc_render.render_cv_html(_tailored_cv()) + skills = html.split("

Skills

")[1].split("")[0] + missing = [s for grp in GRADUATE["skills"] for s in grp["skills"] if s not in skills] + assert missing == [] + + +def test_length_limits_hold_whatever_the_model_returns(): + cv = _full_cvdoc("experienced") + cv["summary"] = " ".join(_sentence(20, i) + "." for i in range(6)) + cv["experience"] += [{"title": f"Old role {r}", "start_date": "2001", "end_date": "2003", + "bullets": ["x"] * 6} for r in range(2)] + cv["experience"][0]["bullets"] += ["Fifth bullet", "Sixth bullet"] + cv["projects"] += [{"name": "Project 3"}, {"name": "Project 4"}] + cv["projects"][0]["description"] = " ".join(_sentence(30, i) + "." for i in range(8)) + cv["skills"][0]["items"] += ["Overflow skill"] + cv["education"] *= 6 + cv["certifications"] *= 3 + + data = doc_render.build_cv_template_data(cv) + assert [r["title"] for r in data["experience"]] == ["Engineer 3", "Engineer 2", "Engineer 1", "Engineer 0"] + assert max(len(r["bullets"]) for r in data["experience"]) == 4 + assert len(data["projects"]) == 3 and len(data["education"]) == 4 and len(data["certifications"]) == 6 + assert len(data["skills"][0]["items"]) == 14 + assert len(data["summary"]) <= 420 and data["summary"].endswith(".") # ends on a whole sentence + assert len(data["projects"][0]["description"]) <= 520 and data["projects"][0]["description"].endswith(".") + + +def test_text_is_escaped_not_rendered_as_markup(): + cv = {"name": "Sam ", + "experience": [{"title": "Intern", "bullets": ["Cut p99 latency to <50ms & halved cost"]}]} + html = doc_render.render_cv_html(cv) + assert "