From 4299cfcfd34ec099aad22e0d633d1521b4491f3c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 06:08:59 +0000 Subject: [PATCH] Report churn in changed characters alongside line counts Line-level blame over-credits one-character fixes (#4). Each pair now carries chars_changed (SequenceMatcher opcodes over the paired lines; unpaired additions and deletions count the full line), each commit row prose_chars_added/prose_chars_deleted over prose lines, and each document prose_char_churn_by_tier - so F1 can be read per 1,000 characters as well as per 1,000 lines. Compared study quantities are unchanged (regression green on the pinned checkout). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MhveTMPqgH3AhmJui3h41m --- CHANGELOG.md | 3 +++ README.md | 2 +- docs/method.md | 4 ++-- src/textstrata/pairs.py | 11 +++++++++++ src/textstrata/scan.py | 34 +++++++++++++++++++++++++++------- tests/test_units.py | 10 +++++++++- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c56fa87..477dbff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,3 +8,6 @@ rejected instead of silently scanning with `script-jump`. - README and `docs/method.md` no longer claim character-level churn is reported; that work is tracked in [#4](https://github.com/QuantEcon/textstrata/issues/4). +- Character-level churn ([#4](https://github.com/QuantEcon/textstrata/issues/4)): `chars_changed` per pair, + `prose_chars_added`/`prose_chars_deleted` per commit and `prose_char_churn_by_tier` per document, so F1 + can be read per 1,000 characters as well as per 1,000 lines. diff --git a/README.md b/README.md index 3dbeda3..c097fab 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ One YAML file per target repository — see [docs/configuration.md](docs/configu ## Limits worth knowing - Squash merges hide human cleanup done inside a machine-drafted PR: `ai-initial` means *as landed*, so human effort is a **lower bound**. -- Blame credits the last toucher: a one-character fix claims the whole line, so human shares are an **upper bound** at line granularity. Churn in changed characters, alongside the line counts, is planned ([#4](https://github.com/QuantEcon/textstrata/issues/4)). +- Blame credits the last toucher: a one-character fix claims the whole line, so human shares are an **upper bound** at line granularity. Churn is therefore also reported in changed characters, where a one-character fix counts as one character. - Pairing lines inside rewritten paragraphs is heuristic; category counts are indicative, not exact. - Latin-script targets (e.g. French from English) have no script signal; the `source-diff` prose strategy for them is planned, not implemented. diff --git a/docs/method.md b/docs/method.md index 9f9dbed..7865986 100644 --- a/docs/method.md +++ b/docs/method.md @@ -39,7 +39,7 @@ All stock metrics count **lines containing the target script**. On raw lines mos | S2 | Baseline survival | S1's `ai-initial` share; plus `difflib` similarity of the initial translation to HEAD | | S3 | Derived review state | `machine-only` (no prose-changing roster commit since translation) → `human-touched` → `audit-stale` (≥ `review_state.stale_after_syncs` machine syncs since the last touch) | | S4 | Freshness | source commits since the state file's `source-sha` (only with a source repo and state directory) | -| F1 | Human churn | prose lines added + deleted by roster-tier commits, per document; normalise by machine-delivered lines and stratify by engine version downstream | +| F1 | Human churn | prose lines added + deleted by roster-tier commits, per document — also in changed characters; normalise by machine-delivered lines or characters and stratify by engine version downstream | | F2 | Overwrites | for each `ai-sync` commit, the prose lines it deleted, blamed at the parent, counted by prior tier | | F3 | Edit categories | pair counts by category and taxonomy bucket | | F4 | Recurring substitutions | short `(before, after)` replacements inside terminology / fluency / width pairs, counted across the corpus | @@ -49,7 +49,7 @@ All stock metrics count **lines containing the target script**. On raw lines mos ## Known limits - **Squash merges** hide human work done inside a machine-drafted PR. `ai-initial` means *as landed*; human effort is a lower bound. -- **Last-toucher blame** credits a whole line to whoever changed one character of it. Human shares are an upper bound at line granularity; reporting churn in changed characters is planned ([#4](https://github.com/QuantEcon/textstrata/issues/4)) — until then the before/after text in `pairs.jsonl` is the only character-level signal. +- **Last-toucher blame** credits a whole line to whoever changed one character of it. Human shares are an upper bound at line granularity; churn is therefore also reported in changed characters (`chars_changed` per pair, `prose_chars_added`/`prose_chars_deleted` per commit, `prose_char_churn_by_tier` per document), where a one-character fix counts as one character. Counts come from `SequenceMatcher` opcodes over the paired lines' raw text; unpaired additions and deletions count the full line. - **Line pairing** inside rewritten paragraphs is heuristic (similarity-matched within a hunk). Category counts are indicative. - **Identity** is resolved by e-mail and GitHub noreply handle only; display names are ignored. Unresolved authors fall to `ai-assisted` and should be reviewed in `commits.jsonl`. - **Pre-engine history** has no recorded engine version. Stratify flow metrics by version downstream and label the pre-engine stratum as such; do not read its rates as the shipping engine's. diff --git a/src/textstrata/pairs.py b/src/textstrata/pairs.py index 422c269..feda159 100644 --- a/src/textstrata/pairs.py +++ b/src/textstrata/pairs.py @@ -83,6 +83,17 @@ def categorise(old: str, new: str, prose: Prose) -> tuple[str, float]: return "retranslation", sim +def changed_chars(old: str, new: str) -> tuple[int, int]: + """(deleted, added) character counts between paired lines, from SequenceMatcher opcodes.""" + dels = adds = 0 + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, old, new).get_opcodes(): + if tag in ("replace", "delete"): + dels += i2 - i1 + if tag in ("replace", "insert"): + adds += j2 - j1 + return dels, adds + + def line_pairs(h: Hunk, prose: Prose) -> tuple[list[tuple[int, str, str]], list[str], list[tuple[int, str]]]: """Pair old/new lines within a hunk. Returns (pairs[(old_lineno, old, new)], adds, dels[(old_lineno, old)]).""" old, new = h.old, h.new diff --git a/src/textstrata/scan.py b/src/textstrata/scan.py index 0515579..996bd61 100644 --- a/src/textstrata/scan.py +++ b/src/textstrata/scan.py @@ -33,7 +33,7 @@ ls_files, show, ) -from .pairs import CATEGORY_MAP, categorise, line_pairs, mine_substitutions, parse_hunks +from .pairs import CATEGORY_MAP, categorise, changed_chars, line_pairs, mine_substitutions, parse_hunks from .prose import Prose from .roster import Roster from .tiers import HUMAN_TIERS, TierContext @@ -56,6 +56,7 @@ class DocResult: similarity_initial_head: float | None = None commits_by_tier: dict[str, int] = field(default_factory=dict) prose_churn_by_tier: dict[str, int] = field(default_factory=dict) # prose lines added+deleted + prose_char_churn_by_tier: dict[str, int] = field(default_factory=dict) # prose characters added+deleted first_human_touch: str | None = None last_human_touch: str | None = None days_to_first_human_touch: int | None = None @@ -122,7 +123,8 @@ def scan(cfg: Config, out_dir: Path, log=sys.stderr) -> dict: tier_of[(f, c.sha)] = tier commit_rows.append({"document": f, "sha": c.sha, "author": c.author, "email": c.email, "date": c.date, "subject": c.subject, "tier": tier, - "adds": c.adds, "dels": c.dels, "prose_adds": 0, "prose_dels": 0}) + "adds": c.adds, "dels": c.dels, "prose_adds": 0, "prose_dels": 0, + "prose_chars_added": 0, "prose_chars_deleted": 0}) d = DocResult(path=f, translated=t_sha is not None, translation_sha=t_sha, translation_date=t_date, n_commits=len(hist)) docs[f] = d @@ -171,6 +173,7 @@ def tier_for(f: str, sha: str) -> str: d.commits_by_tier = dict(Counter(tier_of[(f, c.sha)] for c in post)) # per-commit diffs: prose churn, pairs, overwrites churn: Counter = Counter() + char_churn: Counter = Counter() for c in post: tier = tier_of[(f, c.sha)] try: @@ -180,8 +183,21 @@ def tier_for(f: str, sha: str) -> str: p_adds = sum(1 for h in hunks for ln in h.new if prose.is_prose(ln)) p_dels = sum(1 for h in hunks for ln in h.old if prose.is_prose(ln)) churn[tier] += p_adds + p_dels + # and in characters, so a one-character fix is not credited with the whole line + paired = [line_pairs(h, prose) for h in hunks] + pc_adds = pc_dels = 0 + for pairs, adds, dels in paired: + for _ln, o, n in pairs: + if prose.is_prose(o) or prose.is_prose(n): + d_chars, a_chars = changed_chars(o, n) + pc_dels += d_chars + pc_adds += a_chars + pc_adds += sum(len(x) for x in adds if prose.is_prose(x)) + pc_dels += sum(len(x) for _ln, x in dels if prose.is_prose(x)) + char_churn[tier] += pc_adds + pc_dels row = rows_by_key[(f, c.sha)] row["prose_adds"], row["prose_dels"] = p_adds, p_dels + row["prose_chars_added"], row["prose_chars_deleted"] = pc_adds, pc_dels pr = PR_RE.search(c.subject) if tier == "ai-sync": # what did the machine replace? blame the deleted prose lines at the parent @@ -210,16 +226,17 @@ def tier_for(f: str, sha: str) -> str: continue if tier == "seed": continue - for h in hunks: - pairs, adds, dels = line_pairs(h, prose) + for pairs, adds, dels in paired: for _ln, o, n in pairs: if o.strip() == n.strip(): continue cat, sim = categorise(o, n, prose) + d_chars, a_chars = changed_chars(o, n) pairs_out.append({"document": f, "sha": c.sha[:8], "date": c.date[:10], "tier": tier, "pr": pr.group(1) if pr else None, "category": cat, "taxonomy": CATEGORY_MAP[cat], - "similarity": round(sim, 3), "before": o, "after": n}) + "similarity": round(sim, 3), "chars_changed": d_chars + a_chars, + "before": o, "after": n}) if tier in HUMAN_TIERS and cat in ("terminology", "punctuation-width", "fluency"): mine_substitutions(o, n, prose, subs, sub_examples) for n in adds: @@ -227,14 +244,17 @@ def tier_for(f: str, sha: str) -> str: pairs_out.append({"document": f, "sha": c.sha[:8], "date": c.date[:10], "tier": tier, "pr": pr.group(1) if pr else None, "category": "addition", "taxonomy": "omission", - "similarity": 0.0, "before": "", "after": n}) + "similarity": 0.0, "chars_changed": len(n), + "before": "", "after": n}) for _ln, o in dels: if o.strip(): pairs_out.append({"document": f, "sha": c.sha[:8], "date": c.date[:10], "tier": tier, "pr": pr.group(1) if pr else None, "category": "deletion", "taxonomy": "omission", - "similarity": 0.0, "before": o, "after": ""}) + "similarity": 0.0, "chars_changed": len(o), + "before": o, "after": ""}) d.prose_churn_by_tier = dict(churn) + d.prose_char_churn_by_tier = dict(char_churn) # a human "touch" is a roster-tier commit that changed prose (technical fixes do not count) humans = [c for c in post if tier_of[(f, c.sha)] in HUMAN_TIERS and (rows_by_key[(f, c.sha)]["prose_adds"] + rows_by_key[(f, c.sha)]["prose_dels"]) diff --git a/tests/test_units.py b/tests/test_units.py index 96bb2a3..0688a5c 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -2,7 +2,7 @@ from textstrata.config import Config, ConfigError, ProseConfig, load_config from textstrata.git import Commit, parse_trailers -from textstrata.pairs import categorise, line_pairs, parse_hunks +from textstrata.pairs import categorise, changed_chars, line_pairs, parse_hunks from textstrata.prose import Prose from textstrata.roster import Person, Roster from textstrata.tiers import TierContext @@ -31,6 +31,14 @@ def test_categorise(): assert categorise("完全不同的一句话，没有任何重叠。", "这里讨论价格水平如何决定。", p)[0] == "retranslation" +def test_changed_chars(): + # a one-character punctuation fix counts one character each way, not the line + assert changed_chars("首先解释增长事实是主要目的.", "首先解释增长事实是主要目的。") == (1, 1) + assert changed_chars("这些代理人都居住。", "这些个体都居住。") == (3, 2) + assert changed_chars("一样的行。", "一样的行。") == (0, 0) + assert changed_chars("", "新增的一行。") == (0, 6) + + def test_hunks_and_pairs(): diff = "@@ -10,2 +10,2 @@\n-旧的第一行。\n-旧的第二行。\n+新的第一行。\n+新的第二行。\n@@ -20 +20,0 @@\n-被删除的一行。\n" hunks = parse_hunks(diff)