From da710a17573ea54475c6d96fe4ba634791fb82cc Mon Sep 17 00:00:00 2001 From: S Ravi Kumar Date: Mon, 7 Sep 2026 16:30:09 +0530 Subject: [PATCH 1/2] Fixed the metrics collection --- .tfcore/telemetry/SCHEMA.md | 2 +- .tfcore/telemetry/tf-metrics.sh | 83 +++++- docs/CHANGELOG.html | 32 +++ docs/CHANGELOG.md | 37 +++ docs/TechieFlow-Misses.html | 12 +- docs/TechieFlow-Misses.md | 8 +- docs/TechieFlow-Requirements.md | 10 +- docs/metrics/METRICS.html | 460 ++++++++++++++++++++++++++++++++ docs/metrics/METRICS.md | 157 +++++++++++ docs/metrics/commits.jsonl | 1 + docs/metrics/gates.jsonl | 12 + docs/metrics/misses.jsonl | 3 + docs/metrics/runs.jsonl | 3 + tests/mirror/run.sh | 5 + tests/requirements/run.sh | 122 +++++++++ 15 files changed, 930 insertions(+), 17 deletions(-) create mode 100644 docs/metrics/METRICS.html create mode 100644 docs/metrics/METRICS.md create mode 100644 tests/requirements/run.sh diff --git a/.tfcore/telemetry/SCHEMA.md b/.tfcore/telemetry/SCHEMA.md index e52e59e..6126759 100644 --- a/.tfcore/telemetry/SCHEMA.md +++ b/.tfcore/telemetry/SCHEMA.md @@ -173,7 +173,7 @@ Failure: |---|---|---| | `run_id` | string | The `started` timestamp of the owning run. Ties every REQ verdict in one verify pass together. | | `req_id` | string | e.g. `REQ-UI-004`. | -| `req_class` | string | `UI` \| `FN` \| `RAG` \| `NFR` — the prefix segment of `req_id`. | +| `req_class` | string | `UI` \| `FN` \| `RAG` \| `NFR` — the prefix segment of `req_id`. **`FR` added 2026-09-07**: the framework's own requirement lines (`docs/TechieFlow-Requirements.md`), graded by `tests/requirements/run.sh`. It is the one `req_class` that does not come from an application's checklist, and it never pools with the others — a framework line and a screen requirement are not the same unit. | | `attempt` | int | See §3.1. Derive it; never guess it. | | `verdict` | string | Mirrors the checklist vocabulary **exactly**: `Verified` \| `Needs re-verify` \| `FAIL` \| `Blocked` \| `Implemented` \| `Done (pre-existing)`. | | `gate` | string \| null | **The FIRST gate that failed**, or `null` on a pass. See §3.2. | diff --git a/.tfcore/telemetry/tf-metrics.sh b/.tfcore/telemetry/tf-metrics.sh index b0b669b..43dad1e 100644 --- a/.tfcore/telemetry/tf-metrics.sh +++ b/.tfcore/telemetry/tf-metrics.sh @@ -34,6 +34,7 @@ import re import subprocess import sys from collections import Counter, defaultdict, OrderedDict +from datetime import datetime STREAMS = ("runs", "gates", "sessions", "commits", "misses") VERDICTS = ("Verified", "Needs re-verify", "FAIL", "Blocked", "Implemented", "Done (pre-existing)") @@ -496,6 +497,34 @@ def analyse_phases(runs): when someone asks how it was measured.""" live = [r for r in runs if r.get("kind", "run") == "run" and not r.get("backfilled")] + # A run that carries both ends but no `duration_s` was silently worth ZERO TIME here, + # while its tokens still counted — so a phase's total time was a sum over some of its + # runs and its tokens a sum over others (found 2026-09-07: the reset's own 13 runs + # reported 16h49m, because 6 of them predate the field and the true figure is ~55h). + # Deriving it is arithmetic on two recorded facts, not a guess, so it is done — and + # counted, so the report can say how many of its minutes were derived rather than read. + # `ended` itself may be absent on an older record; SCHEMA.md's own rule is that `ended` + # IS the moment the record was written, which is exactly what `ts` holds. + def _secs(a, b): + try: + fmt = "%Y-%m-%dT%H:%M:%SZ" + return int((datetime.strptime(b, fmt) - datetime.strptime(a, fmt)).total_seconds()) + except Exception: + return None + + derived_n = 0 + for r in live: + if r.get("duration_s") or not r.get("started"): + continue + end = r.get("ended") or r.get("ts") + if not end: + continue + d = _secs(r["started"], end) + if d is not None and d >= 0: + r["duration_s"] = d + r["duration_derived"] = "ended" if r.get("ended") else "ts" + derived_n += 1 + def scope_of(r): return r.get("tokens_scope") or "absent" @@ -564,6 +593,10 @@ def analyse_phases(runs): "median": median(durs), "max": max(durs) if durs else None, "n": len(durs), + # how many of those minutes were computed from the record's own two + # timestamps because the record predates `duration_s` — arithmetic on + # recorded facts, but the reader is told rather than left to assume. + "derived_n": sum(1 for r in rs if r.get("duration_derived")), }, "share_of_duration": pct(sum(durs), grand_dur), "tokens_measured_n": len(priced), @@ -581,6 +614,27 @@ def analyse_phases(runs): key=lambda kv: (-kv[1]["tokens_out"], kv[0]))), "harnesses": OrderedDict(sorted(Counter(r.get("harness") or "?" for r in rs).items())), "modes": OrderedDict(sorted(Counter(r.get("mode") or "—" for r in rs).items())), + # A phase run in named modes (a reset's sessions, a build's fresh/fix passes) + # is several different jobs under one `cmd`. The per-mode split says which, + # in the order they happened. Same denominators as the phase above: a run + # with no computable window contributes no tokens, never a zero. + "by_mode": OrderedDict( + (mode, { + "runs": len([r for r in rs if (r.get("mode") or "—") == mode]), + "duration_s": sum(r.get("duration_s") or 0 + for r in rs if (r.get("mode") or "—") == mode), + "tokens_out": sum(r.get("tokens_out") or 0 + for r in rs if (r.get("mode") or "—") == mode and has_tokens(r)), + "tokens_unmeasured_n": len([r for r in rs + if (r.get("mode") or "—") == mode and not has_tokens(r)]), + "files_written": sum(r.get("files_written") or 0 + for r in rs if (r.get("mode") or "—") == mode), + "first_started": min((r.get("started") or "" for r in rs + if (r.get("mode") or "—") == mode), default=""), + }) + for mode in sorted({(r.get("mode") or "—") for r in rs}, + key=lambda m: min((r.get("started") or "" for r in rs + if (r.get("mode") or "—") == m), default=""))), "build_result": OrderedDict(sorted(Counter(r.get("build_result") or "—" for r in rs).items())), "reqs_touched_total": sum(r.get("reqs_count") or 0 for r in rs), "files_written_total": sum(r.get("files_written") or 0 for r in rs), @@ -951,6 +1005,20 @@ def analyse(repos): live = [g for g in gates if not g.get("backfilled")] back = [g for g in gates if g.get("backfilled")] + # `attempt` is DEFINED by §3.1 as 1 + the number of prior live gate records for the same + # requirement in the same app — a count over this very stream, not a judgement. A record + # that omits it was therefore being read as "no attempt", which drops it out of the + # first-pass rate and reports 0% for a set of records that all passed on their first and + # only verdict (found 2026-09-07 on the framework's own requirement lines). Derived here, + # in stream order, for records that lack it. A record that carries one is never touched. + _seen = {} + for g in sorted(live, key=lambda r: (r.get("ts") or "", r.get("run_id") or "")): + key = (g.get("app"), g.get("req_id")) + _seen[key] = _seen.get(key, 0) + 1 + if g.get("attempt") is None: + g["attempt"] = _seen[key] + g["attempt_derived"] = True + # REQs with ANY backfilled record are excluded from the live first-pass rate: # their live `attempt` numbering restarts at 1 (SCHEMA.md §3.1). # A requirement is keyed by (app, req_id): REQ-UI-001 exists in every project, so a rollup that @@ -1192,8 +1260,9 @@ def print_phases(p, W): for cmd, m in p["phases"].items(): print(" [%s] %d run(s)" % (cmd, m["runs"])) d = m["duration_s"] - print(" wall clock : total %s · median %s · max %s (n=%d timed)" - % (_hms(d["total"]), _hms(d["median"]), _hms(d["max"]), d["n"])) + print(" wall clock : total %s · median %s · max %s (n=%d timed%s)" + % (_hms(d["total"]), _hms(d["median"]), _hms(d["max"]), d["n"], + ", %d derived from the record's own timestamps" % d["derived_n"] if d.get("derived_n") else "")) t = m["tokens"] print(" tokens : out %s · in %s · cache-read %s · cache-write %s" % (_k(t["out"]), _k(t["in"]), _k(t["cache_read"]), _k(t["cache_write"]))) @@ -1255,7 +1324,15 @@ def print_phases(p, W): print(" cost (%s) : $%s over %d record(s) — MEASURED, never pooled " "across harness" % (h, c["usd"], c["records"])) if m["reqs_touched_total"] or m["files_written_total"]: - print(" work : %d REQ touch(es) · %d file(s) written" + bm = m.get("by_mode") or {} + if len(bm) > 1: + print(" by mode : (in the order they ran)") + for mode, v in bm.items(): + unm = (" [%d unmeasured]" % v["tokens_unmeasured_n"]) if v["tokens_unmeasured_n"] else "" + print(" %-22s %2d run(s) %8s %8s out %3d file(s)%s" + % (mode, v["runs"], _hms(v["duration_s"]), _k(v["tokens_out"]), + v["files_written"], unm)) + print(" work : %d REQ touch(es) · %d file(s) written" % (m["reqs_touched_total"], m["files_written_total"])) print("") diff --git a/docs/CHANGELOG.html b/docs/CHANGELOG.html index cb6de6c..06ffed3 100644 --- a/docs/CHANGELOG.html +++ b/docs/CHANGELOG.html @@ -114,6 +114,8 @@

TechieFlow — Changelog

  • The reset (2026-09-04 to 2026-09-07)
  • The 2026-08-28 review, as it stood at the top of the old briefing
  • Maintenance log (newest first)
    • +
  • 2026-09-07 — the framework graded against its own checklist for the first time
    • +
  • 2026-09-07 — the reset measured: docs/metrics/METRICS.md written, and a phase's time was being under-reported
  • 2026-09-07 — the Codex adapter removed, and WORKFLOW.html dropped
  • 2026-09-07 — main merged into dev: the npm installer brought back in step with the shell scripts
  • 2026-09-07 — Session 7: the Playbook review prompt, version 2, and the end of the reset
    • @@ -213,6 +215,36 @@

    The 20

    Maintenance log (newest first)#

    +

    2026-09-07 — the framework graded against its own checklist for the first time#

    +

    The owner rejected the reasoning in the metrics report, and was right. It said the framework had no first-pass rate because it had "no checklist of its own to verify". It has had one since Session 2: docs/TechieFlow-Requirements.md, 63 numbered lines, each with a stated check, named in every session restart prompt. The framework had been demanding of every application a verification it never performed on itself. Logged as MISS-TechieFlow-20260907-16, severity blocker, sorted ignored — the rule was written down and not followed.

    +

    tests/requirements/run.sh now grades it. A line is graded only when its own Check column names something runnable; that artefact is run and its exit status is the verdict. A line whose check is a fixture run, a review, or a script that was never built is reported ungraded with the reason, never guessed.

    + + + + + + + +
    Requirement lines63
    Graded, and passing12
    Ungraded51 — 28 describe a script nobody built · 14 need a fixture run · 5 need a review · 3 unbuilt candidates · 1 needs a non-Windows filesystem
    +

    Those 28 are the real finding: the framework wrote down how it would check itself and then did not build the check. Five lines were repointed at the test that genuinely proves them (FR-14, FR-24, FR-26, FR-43, FR-44), and FR-26 gained a real check — the verify task, once 11,850 words, is now held under 4,000 and measures 964.

    +

    Two reporting defects fell out of it. req_class gained FR in the schema, deliberately and dated, so the framework's own lines can be graded without being pooled with an application's screens. And a gate record with no attempt was invisible to the first-pass rate, which reported 0% for twelve records that had all passed; attempt is defined by the schema as a count over the stream, so it is now derived at read time when absent. Verified not to move any application's published figures: TfLens stays at 20% and 94%, Lekhak at 54%.

    +

    2026-09-07 — the reset measured: docs/metrics/METRICS.md written, and a phase's time was being under-reported#

    +

    The framework had never written its own metrics report. Producing it found that the reset's cost was being under-reported by two thirds.

    +

    A run with both timestamps and no duration_s was worth zero time. Six of the reset's records predate that field, so the report summed time over seven runs while summing tokens over twelve: framework-reset showed 16h49m where the same records already proved 55h57m. The duration is now derived at read time from the record's own started and ended (or from ts, which the schema defines as the moment the record was written), and the report says how many of its minutes were derived rather than read. Logged as MISS-TechieFlow-20260907-15, fixed, and propagated to all 23 projects. The correction also moves the repository's REQ throughput from 1.86 to 1.83 an hour, which is the honest figure.

    +

    A phase can now be broken down by mode, so a reset reads session by session rather than as one 56-hour block. The framework's own numbers, at the close:

    + + + + + + + + + + +
    The reset14 runs, 56h 30m, 7.9M output tokens, 308 files written
    Where it wentthe four Session 4 sittings took 43 of the 56.5 hours and 5.0M of the 7.9M tokens
    Everything after Session 42h 05m across five sessions
    Modelsclaude-fable-5-1 98% of output over 9 runs; claude-opus-5 2% over 4
    Misses121 logged, 33 open, 87 resolved, 1 will-not-fix
    Whose gap, of the 66 sortedcheck too weak 50% · never said 27% · said and ignored 23%
    +

    That last row is the reset's own justification: half of what the framework got wrong was a check too weak to catch it, not a rule nobody had written. Turning prose into scripts was the right treatment.

    +

    Reported honestly rather than filled in: gates.jsonl is empty, so this repository has no first-pass rate, gate distribution or escape rate, and Session 6's own run record carries no token window and is excluded from every token figure instead of counted as zero.

    2026-09-07 — the Codex adapter removed, and WORKFLOW.html dropped#

    Both on the owner's decision, closing D-14 (open since the Session 1 review) and MISS-TechieFlow-20260907-10.

    Codex. The framework supported three harnesses on paper and two in practice. The adapter was frozen through the reset and is now gone: .codex/ and .agents/skills/ deleted here, the binder, the telemetry reader and the adapter hook deleted with them, and every Codex branch taken out of the goal supervisor, the harness resolver, the emitter's harness detection, the routing scripts, routing.yaml, three guard hooks, four templates, the user guide and the telemetry schema. Both delivery routes stopped deploying it and started removing it, so one propagation pass cleaned every project rather than anyone deleting folders by hand. The telemetry schema keeps codex as a retired harness value, because records written before today carry it and a reader must still understand them.

    diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cac057f..b9974d0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,43 @@ Everything below predates the reset and is preserved as it was written. ## Maintenance log (newest first) +### 2026-09-07 — the framework graded against its own checklist for the first time + +The owner rejected the reasoning in the metrics report, and was right. It said the framework had no first-pass rate because it had "no checklist of its own to verify". It has had one since Session 2: **`docs/TechieFlow-Requirements.md`, 63 numbered lines, each with a stated check**, named in every session restart prompt. The framework had been demanding of every application a verification it never performed on itself. Logged as `MISS-TechieFlow-20260907-16`, severity blocker, sorted `ignored` — the rule was written down and not followed. + +**`tests/requirements/run.sh`** now grades it. A line is graded only when its own Check column names something runnable; that artefact is run and its exit status is the verdict. A line whose check is a fixture run, a review, or a script that was never built is reported **ungraded with the reason**, never guessed. + +| | | +|---|---| +| Requirement lines | 63 | +| Graded, and passing | 12 | +| Ungraded | 51 — 28 describe a script nobody built · 14 need a fixture run · 5 need a review · 3 unbuilt candidates · 1 needs a non-Windows filesystem | + +Those 28 are the real finding: the framework wrote down how it would check itself and then did not build the check. Five lines were repointed at the test that genuinely proves them (FR-14, FR-24, FR-26, FR-43, FR-44), and FR-26 gained a real check — the verify task, once 11,850 words, is now held under 4,000 and measures 964. + +**Two reporting defects fell out of it.** `req_class` gained `FR` in the schema, deliberately and dated, so the framework's own lines can be graded without being pooled with an application's screens. And a gate record with no `attempt` was invisible to the first-pass rate, which reported **0% for twelve records that had all passed**; `attempt` is defined by the schema as a count over the stream, so it is now derived at read time when absent. Verified not to move any application's published figures: TfLens stays at 20% and 94%, Lekhak at 54%. + +### 2026-09-07 — the reset measured: `docs/metrics/METRICS.md` written, and a phase's time was being under-reported + +The framework had never written its own metrics report. Producing it found that the reset's cost was being under-reported by two thirds. + +**A run with both timestamps and no `duration_s` was worth zero time.** Six of the reset's records predate that field, so the report summed time over seven runs while summing tokens over twelve: `framework-reset` showed **16h49m** where the same records already proved **55h57m**. The duration is now derived at read time from the record's own `started` and `ended` (or from `ts`, which the schema defines as the moment the record was written), and the report says how many of its minutes were derived rather than read. Logged as `MISS-TechieFlow-20260907-15`, fixed, and propagated to all 23 projects. The correction also moves the repository's REQ throughput from 1.86 to 1.83 an hour, which is the honest figure. + +**A phase can now be broken down by mode**, so a reset reads session by session rather than as one 56-hour block. The framework's own numbers, at the close: + +| | | +|---|---| +| The reset | 14 runs, 56h 30m, 7.9M output tokens, 308 files written | +| Where it went | the four Session 4 sittings took 43 of the 56.5 hours and 5.0M of the 7.9M tokens | +| Everything after Session 4 | 2h 05m across five sessions | +| Models | claude-fable-5-1 98% of output over 9 runs; claude-opus-5 2% over 4 | +| Misses | 121 logged, 33 open, 87 resolved, 1 will-not-fix | +| Whose gap, of the 66 sorted | check too weak 50% · never said 27% · said and ignored 23% | + +That last row is the reset's own justification: **half of what the framework got wrong was a check too weak to catch it, not a rule nobody had written.** Turning prose into scripts was the right treatment. + +Reported honestly rather than filled in: `gates.jsonl` is empty, so this repository has no first-pass rate, gate distribution or escape rate, and Session 6's own run record carries no token window and is excluded from every token figure instead of counted as zero. + ### 2026-09-07 — the Codex adapter removed, and WORKFLOW.html dropped Both on the owner's decision, closing D-14 (open since the Session 1 review) and `MISS-TechieFlow-20260907-10`. diff --git a/docs/TechieFlow-Misses.html b/docs/TechieFlow-Misses.html index 17ef605..9f1fdf1 100644 --- a/docs/TechieFlow-Misses.html +++ b/docs/TechieFlow-Misses.html @@ -111,8 +111,8 @@

    TechieFlow — Misses

    Contents
      -
    1. Open (33)
      2. -
    2. Fixed (86)
      3. +
    3. Open (34)
      4. +
    4. Fixed (87)
    5. Will not fix (1)
    @@ -120,16 +120,17 @@

    TechieFlow — Misses

    AppTechieFlow -Count120 logged: 33 open, 86 fixed, 1 will not fix +Count122 logged: 34 open, 87 fixed, 1 will not fix Sourcedocs/metrics/misses.jsonl, one row per miss record. Rewritten by tf-misses-md.sh on every new record. Never edit it: a wrong row is corrected by a new record. Updated2026-09-07

    Whose gap answers the four questions of the miss protocol: the app's spec did not say it, so the checklist line is fixed; the framework never said it, so one requirement line and a check are added; the check was too weak (a review, or a script that did not fire), so the check is fixed; said and ignored, so the rule becomes a hook or is deleted. not sorted means the record predates the sort or nobody has answered yet; bash .tfcore/utils/tf-emit.sh --amend <miss> sort <spec|unsaid|weak-check|ignored> completes it.

    -

    Open (33)#

    +

    Open (34)#

    + @@ -165,10 +166,11 @@

    Open (33)#

    MissFoundWhose gapWhat went wrong
    MISS-TechieFlow-20260907-162026-09-07 by ownersaid and ignoredThe framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all.
    MISS-TechieFlow-20260907-122026-09-07 by ownersaid and ignoredThe D-21 rule that hidden framework folders are invisible to search was written down and still cost 58 percent of a folder's true size when OpenCode measured the Playbook with a shell glob.
    MISS-TechieFlow-20260907-042026-09-07 by gatethe check was too weakThe readable miss file's unchanged check compared only the header above the Updated line, so an amend that changed a row but no count left the file stale; the bugs self-test caught it before release and the check now compares everything but the date.
    MISS-TechieFlow-20260907-032026-09-07 by agent-reviewthe check was too weakThe cross-project rollup keyed a requirement by its id alone, so REQ-UI-001 of TfLens and REQ-UI-001 of TechieBlog counted as one requirement and the combined first-pass rate printed 72% where the true figure is 48%; found by re-reading the numbers before the explainer, fixed by keying on project and id.
    (no id, record 55)2026-09-05 by ownernot sortedThe first Session 4a task table put the owner questions inside table cells and the rows were too wide to read in a terminal, against the plain-words rule.
    -

    Fixed (86)#

    +

    Fixed (87)#

    + diff --git a/docs/TechieFlow-Misses.md b/docs/TechieFlow-Misses.md index 0883b2b..05f26f3 100644 --- a/docs/TechieFlow-Misses.md +++ b/docs/TechieFlow-Misses.md @@ -3,16 +3,17 @@ | | | |---|---| | App | TechieFlow | -| Count | 120 logged: 33 open, 86 fixed, 1 will not fix | +| Count | 122 logged: 34 open, 87 fixed, 1 will not fix | | Source | `docs/metrics/misses.jsonl`, one row per miss record. Rewritten by `tf-misses-md.sh` on every new record. Never edit it: a wrong row is corrected by a new record. | | Updated | 2026-09-07 | **Whose gap** answers the four questions of the miss protocol: **the app's spec** did not say it, so the checklist line is fixed; **the framework never said it**, so one requirement line and a check are added; **the check was too weak** (a review, or a script that did not fire), so the check is fixed; **said and ignored**, so the rule becomes a hook or is deleted. **not sorted** means the record predates the sort or nobody has answered yet; `bash .tfcore/utils/tf-emit.sh --amend sort ` completes it. -## Open (33) +## Open (34) | Miss | Found | Whose gap | What went wrong | |---|---|---|---| +| MISS-TechieFlow-20260907-16 | 2026-09-07 by owner | said and ignored | The framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all. | | MISS-TechieFlow-20260907-12 | 2026-09-07 by owner | said and ignored | The D-21 rule that hidden framework folders are invisible to search was written down and still cost 58 percent of a folder's true size when OpenCode measured the Playbook with a shell glob. | | MISS-TechieFlow-20260907-04 | 2026-09-07 by gate | the check was too weak | The readable miss file's unchanged check compared only the header above the Updated line, so an amend that changed a row but no count left the file stale; the bugs self-test caught it before release and the check now compares everything but the date. | | MISS-TechieFlow-20260907-03 | 2026-09-07 by agent-review | the check was too weak | The cross-project rollup keyed a requirement by its id alone, so REQ-UI-001 of TfLens and REQ-UI-001 of TechieBlog counted as one requirement and the combined first-pass rate printed 72% where the true figure is 48%; found by re-reading the numbers before the explainer, fixed by keying on project and id. | @@ -47,10 +48,11 @@ | MISS-TechieFlow-20260904-01 | 2026-09-04 by owner | not sorted | no sentence recorded (wrong-behaviour, other, why: instruction-ignored) | | (no id, record 55) | 2026-09-05 by owner | not sorted | The first Session 4a task table put the owner questions inside table cells and the rows were too wide to read in a terminal, against the plain-words rule. | -## Fixed (86) +## Fixed (87) | Miss | Found | Closed | Whose gap | What went wrong | |---|---|---|---|---| +| MISS-TechieFlow-20260907-15 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the check was too weak | A run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved. | | MISS-TechieFlow-20260907-14 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the framework never said it | The npm installer's framework subfolder list left out standards, so a project migrated from the old layout came out with no coding standards file. | | MISS-TechieFlow-20260907-13 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the framework never said it | The npm installer wrote a settings.json missing five hook registrations the shell scripts had gained, so a project installed from the package ran without the metrics, database and build guards. | | MISS-TechieFlow-20260907-11 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the check was too weak | The Playbook review prompt was drafted from folder word counts taken only at the top level, so 174 files and 196,498 words counted as zero and the prompt nearly shipped the wrong headline finding. | diff --git a/docs/TechieFlow-Requirements.md b/docs/TechieFlow-Requirements.md index 2bda419..3691ea0 100644 --- a/docs/TechieFlow-Requirements.md +++ b/docs/TechieFlow-Requirements.md @@ -70,7 +70,7 @@ The same four questions are asked for a miss in an application, against that app | FR-11 | applies mockups to any project, not only greenfield; brownfield day-1 finds existing mockups, records their location, and links them from the documents. | fixture run: `*day1-brownfield Xpenser` with a `docs/mockups/` folder present; the UIDesign and BRD link them and no mockup is regenerated. | D-5 | | FR-12 | produces the checklist automatically once the owner approves the BRD; the owner never types `*split-brd`. | fixture run: after the stage 2 go-ahead on MyDiary the checklist exists without a separate command. | D-6 | | FR-13 | produces the DevGuide automatically when the build phase completes the checklist, for every project type, and refreshes it at handoff. | fixture run: `*build-phase MyDiary` to completion; `docs/MyDiary-DevGuide.md` exists at the end. | D-7 | -| FR-14 | gives every human document template a strict structure (required sections in order, size budget per size class, row rules) and refuses to close a phase whose document breaks it. | script: `tf-doc-check.sh` exits 0 on every document the fixture runs produce, exits non-zero on a deliberately broken one, and the status gate refuses to close. | How-It-Works §2 Template; Session 3 | +| FR-14 | gives every human document template a strict structure (required sections in order, size budget per size class, row rules) and refuses to close a phase whose document breaks it. | script: `bash tests/doc-check/run.sh` — `tf-doc-check.sh` exits 0 on a clean generated document set, non-zero on the deliberately broken twin, and the status gate refuses to close. | How-It-Works §2 Template; Session 3 | | FR-15 | requires every checklist row and every BRD item to carry one acceptance line of the form "when … then …" naming an observable result, of at most 30 words (target 20) and holding one behaviour, under a title in everyday words; BRD items sit under a heading per screen that opens with one plain sentence. | script: every `REQ-` row and `BRD-N` item in the fixture documents matches the pattern and the word cap; the broken twin's 42-word line fails. | D-20; owner 2026-09-06 (miss 13) | | FR-16 | keeps one checklist per application as the single source of truth, in markdown only, and never creates dated `docs/qa/` or `docs/verify/` files or `-v2` document copies. | script: no `docs/qa/`, `docs/verify/`, `*-v2.*` or `*-Checklist.html` in any fixture. | conventions | | FR-17 | renders every human document to HTML by script, never by hand; configuration and agent documents are not rendered. | script: every human `docs/*.md` has a sibling `.html` newer than itself; no `.html` exists for the checklist, the Stack documents or this file. | TF-003; owner 2026-09-04 | @@ -92,9 +92,9 @@ The same four questions are asked for a miss in an application, against that app | ID | The framework … | Check | Source | |---|---|---|---| -| FR-24 | applies the seven checks to every requirement in a fixed order and records the first that fails. | script: every `gates.jsonl` record from a fixture verify carries a gate value from the fixed list or none. | How-It-Works §6.2 | +| FR-24 | applies the seven checks to every requirement in a fixed order and records the first that fails. | script: `bash tests/verify/run.sh` — every `gates.jsonl` record the fixture verify writes carries a gate value from the fixed list or none. | How-It-Works §6.2 | | FR-25 | verifies against the acceptance line and the mockup, and states in the remark what was observed, so a `Verified` row can be re-derived by a reader. | review, to become a script: sample ten verified rows across fixtures; each remark names the observation. | 63 misses classified insufficient-verify-method | -| FR-26 | has a verify task of at most 4,000 words, with every mechanical step in a script. | script: word count of `verify-phase.md` ≤ 4,000. | D-8 | +| FR-26 | has a verify task of at most 4,000 words, with every mechanical step in a script. | script: `bash tests/mirror/run.sh` counts `verify-phase.md` and fails above 4,000 words (built 2026-09-07; it is 964). | D-8 | | FR-27 | never reports a file or tool as "not present" without trying its literal path, because the framework folder is invisible to search. | script: the phrase "not present" in a checklist remark is refused unless the remark also names the path tried. | D-21 | ### E. Bugs and misses @@ -138,8 +138,8 @@ The same four questions are asked for a miss in an application, against that app | ID | The framework … | Check | Source | |---|---|---|---| -| FR-43 | reads no more than the instruction budget before the first useful step of any command. The budget is a variable per model tier in `routing.yaml`, not a fixed number: 7,000 words for the frontier tier (about 9,500 tokens, under 5 percent of a 200,000-token context), and smaller for the standard and economy tiers. A task is written as a short core plus reference sections loaded only when a step needs them, so a small budget can be met without losing steps. | script: the section 5 table of `TechieFlow-How-It-Works.md` recomputed per tier; every "at start" value ≤ that tier's budget. | How-It-Works §5; owner question 2026-09-04 | -| FR-44 | keeps the shared rule files under 3,000 words in total and every persona under 1,500. | script: word counts. | How-It-Works §5 | +| FR-43 | reads no more than the instruction budget before the first useful step of any command. The budget is a variable per model tier in `routing.yaml`, not a fixed number: 7,000 words for the frontier tier (about 9,500 tokens, under 5 percent of a 200,000-token context), and smaller for the standard and economy tiers. A task is written as a short core plus reference sections loaded only when a step needs them, so a small budget can be met without losing steps. | script: `bash tests/mirror/run.sh` fails when any task file passes the 7,000-word frontier budget. The per-tier recomputation of the §5 table remains a review. | How-It-Works §5; owner question 2026-09-04 | +| FR-44 | keeps the shared rule files under 3,000 words in total and every persona under 1,500. | script: `bash tests/mirror/run.sh` fails when the shared rule files together pass 3,000 words or any persona passes 1,500. | How-It-Works §5 | | FR-45 | keeps explanation and history out of task files; a task file contains steps only. | review, to become a script: no paragraph in a task file begins with "Why", "Because", "This exists", or a date. | How-It-Works §7 | | FR-46 | converts a prose rule to a hook or deletes it after the second recorded `instruction-ignored` miss against it. | script: the miss report lists prose rules with two or more `instruction-ignored` misses; the list is empty. | §3 question 4 | | FR-47 | names only public repositories in the documents a person outside the owner's machines reads: the README, the briefing, and every template that ships into a project. The reset's own working documents name fixtures on purpose and are out of scope until the owner rules otherwise. | script (built 2026-09-07): `bash tests/mirror/run.sh` reads the owner's private names from `~/.techieflow/private-names.txt`, a per-machine file that is in no repository, and fails when one appears in the README, `WorkFlow-Context.md` or a template. Without that file the check says it was skipped. Reworded from MISS-TechieFlow-20260907-05: the old check named a script that was never written, and a private project sat in the public README for months. | owner 2026-09-04 | diff --git a/docs/metrics/METRICS.html b/docs/metrics/METRICS.html new file mode 100644 index 0000000..4e082d2 --- /dev/null +++ b/docs/metrics/METRICS.html @@ -0,0 +1,460 @@ + + + + + +TechieFlow — Development Metrics + + + + + +
    + +
    +

    TechieFlow — Development Metrics

    +
    Rendered 2026-09-07 · source METRICS.md
    + +
    +
    Contents
    +
      +
    1. 1. First-pass rate — the framework's own requirements
      2. +
    2. 2. Gate catch distribution
      3. +
    3. 3. Escape rate
      4. +
    4. 4. Throughput and rework — poolable
      5. +
    5. 5. Misses — what was missed, who missed it, what the fix cost
      6. +
    6. 5a. Attribution — linked records only
      7. +
    7. 5b. Rework cost — measured and apportioned never combine
      8. +
    8. 6. Effort per phase — time, tokens, model, fan-out
      9. +
    9. 6a. The reset, session by session
      10. +
    10. 6b. Which model did the work
      11. +
    11. 6c. Subagent fan-out — measured, on its own denominator
      12. +
    12. 7. What is missing
      13. +
    +
    +

    Snapshot as of 2026-09-07 · project_type framework · schema v1

    +

    This repository is the framework itself, not an application. It does have a checklist of its own — docs/TechieFlow-Requirements.md, 63 numbered lines each with a stated way to prove it — and since 2026-09-07 it is graded against them by tests/requirements/run.sh like any other project. What it has no data for is a screen verdict: it has no application to boot, so the gate catch distribution and escape rate below stay empty for a reason that is about screens, not about requirements. Alongside that it measures the cost of maintaining the framework, which in this snapshot is almost entirely the seven-session reset of 2026-09-04 to 2026-09-07.

    +
    MissFoundClosedWhose gapWhat went wrong
    MISS-TechieFlow-20260907-152026-09-07 by owner2026-09-07 by fix-issuesthe check was too weakA run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved.
    MISS-TechieFlow-20260907-142026-09-07 by owner2026-09-07 by fix-issuesthe framework never said itThe npm installer's framework subfolder list left out standards, so a project migrated from the old layout came out with no coding standards file.
    MISS-TechieFlow-20260907-132026-09-07 by owner2026-09-07 by fix-issuesthe framework never said itThe npm installer wrote a settings.json missing five hook registrations the shell scripts had gained, so a project installed from the package ran without the metrics, database and build guards.
    MISS-TechieFlow-20260907-112026-09-07 by owner2026-09-07 by fix-issuesthe check was too weakThe Playbook review prompt was drafted from folder word counts taken only at the top level, so 174 files and 196,498 words counted as zero and the prompt nearly shipped the wrong headline finding.
    + + + + + + + + +
    StreamRecordsSpan
    runs.jsonl422026-08-28 → 2026-09-07
    gates.jsonl122026-09-07 (the framework's own requirement lines, graded for the first time)
    sessions.jsonl292026-08-20 → 2026-09-07
    commits.jsonl522026-06-25 → 2026-09-07
    misses.jsonl121 miss + 88 miss-fix + 53 miss-amend2026-08-28 → 2026-09-07
    +
    +

    1. First-pass rate — the framework's own requirements#

    +

    100% of the 12 lines that can be graded, 12 of 63. Every one passed on its first recorded verdict.

    +

    That number needs its denominator and its date, or it flatters the framework:

    + + + + + + + + + +
    Requirement lines in docs/TechieFlow-Requirements.md63
    Graded by a check that runs today12
    Passed12
    Failed0
    Ungraded51
    +

    Why only 12. A line is graded only when its own Check column names something runnable — a self-test, a script, an npm script — and that artefact is run for the verdict. Of the other 51: 14 name a fixture run (a real command on a real project, which no automated pass can stand in for), 5 name a review by a person, 1 needs a normal filesystem (FR-63, which a Windows mount cannot grade honestly), 3 are script candidates that were never written, and 28 describe a script in prose that has no runnable artefact behind it. Those 28 are the finding: the framework wrote down how it would check itself and then did not build the check.

    +

    What this figure is not. These 12 lines were graded for the first time on 2026-09-07, at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: 121 misses logged during the same period (§5). Read the two together or neither.

    +

    Until 2026-09-07 this section read "no data", on the reasoning that the framework had no checklist to verify. That was wrong: it has had 63 requirement lines since Session 2, named in every session restart prompt. Logged as MISS-TechieFlow-20260907-16.

    +

    2. Gate catch distribution#

    +

    No data, and here the original reasoning does hold: a gate distribution answers "which of the seven checks caught the failure", and those checks — build, acceptance, data, visual, assets, speed, standards — are applied to a running application's screens. This repository has none. All 12 framework verdicts passed, so there is no failure to attribute in any case.

    +

    Nothing is inferred from the miss stream to fill the gap: the two are computed from different records by different definitions, and presenting one as the other would make the word meaningless.

    +

    3. Escape rate#

    +

    No data from gates.jsonl: an escape is a defect that got past every gate to a person, and with no screens to gate there is nothing for one to escape. The miss stream's own "found by a human" share — 41% — is reported in §5 beside this, never merged into it. For the framework that share is the meaningful number, and it says the owner found two of every five framework defects.

    +

    4. Throughput and rework — poolable#

    + + + + + + + + + + + +
    FigureValueNote
    Runs recorded42framework-reset 14 · log-miss 22 · fix-issues 6
    Rework ratioinsufficient datano build-phase runs in this repository
    Batch sizeinsufficient datasame reason
    REQ throughput1.83 REQ/hourmedian across runs; here a "REQ" is a framework requirement line (FR-nn) touched by a maintenance run
    Sessions and tokens29 sessions · 10,415,290 tokens3 duplicate session ids collapsed, normal for OpenCode
    Commit cadence2.17 commits per active day52 commits over 24 active days
    Tokens per Verifiedinsufficient datanothing is verified in this repository
    +

    5. Misses — what was missed, who missed it, what the fix cost#

    +

    121 misses logged: 33 open, 87 resolved, 1 will-not-fix, with 88 fix records. Will-not-fix is a decision, not a backlog item, so it is not counted as open.

    + + + + + + + + +
    CutDistribution
    Classwrong-behaviour 64 (53%) · unspecified-gap 26 (21%) · partial-implementation 17 (14%) · scope-creep 6 (5%) · other 3 · spec-contradiction 3 · missed-requirement 2
    Why it was missed (118 of 121 assessed)insufficient-verify-method 46 (39%) · missing-checklist-item 40 (34%) · instruction-ignored 26 (22%) · ambiguous-acceptance 5 (4%) · other 1
    Whose gap (66 of 66 sorted)weak-check 33 (50%) · unsaid 18 (27%) · ignored 15 (23%)
    Found byagent-review 52 · owner 50 · library-feedback 11 · gate 8
    + +

    The headline finding of the "whose gap" cut: half of what the framework got wrong was a check that was too weak, not a rule nobody had written. That is why the reset's method was to turn prose into scripts rather than to add prose.

    +

    5a. Attribution — linked records only#

    +

    5 of 121 records (116 excluded as inferred or unknown.) An excluded record named a phase that no run record backs, so its model is unknown, and a per-model rate computed from guesses is a routing decision made on invented evidence.

    + +

    At n=5 this supports no per-model conclusion and none is drawn. A per-model miss rate is observational, not causal: which model gets the hard work is not random.

    +

    5b. Rework cost — measured and apportioned never combine#

    + + + + + + + +
    AttributionFix recordsTokens out per miss
    sole — measured4289,563 (n=4 priced)
    shared — apportioned by equal division, not a measurement6381,140 (n=63)
    unattributable — no usable token window20not costed
    +

    Four further records stored as none do have a measured window; the divisor is recomputed at read time from the misses each run actually closed. No dollar figure exists: Claude Code carries cost_usd: null permanently and is never priced from a rate card. The only measured money in this repository is $0.230819 across one OpenCode log-miss run.

    +

    6. Effort per phase — time, tokens, model, fan-out#

    +

    42 live run records. Token-window coverage: main 28 · none 7 · conversation 3 · tree 1 · absent 1. A window is only as good as its scope, and a run whose window could not be computed is excluded from every token figure rather than averaged in as a zero.

    + + + + + + + +
    PhaseRunsWall clockTokens outTokens in% out% time
    framework-reset1456h 30m7.9M60.8k95%94%
    fix-issues63h 05m418.7k6.6M5%5%
    log-miss2229m 45s30.4k2.9M0%1%
    +

    framework-reset costing more than log-miss is a fact about what those phases are, not a finding about either.

    +

    6a. The reset, session by session#

    +

    The framework's own maintenance, in the order it ran. Time on six of these fourteen records was computed from the record's own two timestamps, because they predate the duration_s field; that is arithmetic on recorded facts, and it is flagged here rather than left to assume.

    + + + + + + + + + + + + + + + +
    SessionRunsWall clockTokens outFiles written
    Sessions 1 to 3 (before the mode field)310h 56m1.7M48
    Sitting 4a19h 55m1.0M2
    Sitting 4b324h 38m2.6M104
    Sitting 4c18h 21m1.4M42
    Session 5153m 53s989.5k24
    Session 6130m 19sunmeasured34
    Session 7120m 37s100.7k4
    Session 7, merge fix118m 24s39.4k5
    Codex removal12m 35s4.5k41
    This metrics pass132m 51s45.5k4
    Total1456h 30m7.9M over 13 records308
    +

    Session 6's own record carries no token window and is excluded from every token figure above rather than counted as zero. That gap is itself a recorded defect (MISS-TechieFlow-20260907-09): the emitter accepted a run record with no ended, so the run could never be costed. It is fixed, and the fix is what makes the other twelve rows complete.

    +

    Reading the shape: the four Session 4 sittings account for 43 of the 56.5 hours and 5.0M of the 7.9M output tokens. That was the work of shrinking every task file, and it cost roughly three quarters of the whole reset. The five sessions that followed — the miss protocol, the readability split, the Playbook prompt, the merge fix and the Codex removal — took two hours and five minutes between them.

    +

    6b. Which model did the work#

    + + + + + + + +
    ModelTokens outRunsShare
    claude-fable-5-17.7M998%
    claude-opus-5190.1k42%
    synthetic (no model recorded)020%
    +

    Harness: claude-code on all 14 reset runs. Model routing across the whole repository was observed as on-tier 0 · drifted 10 · unknown 12 — observed, never enforced.

    +

    6c. Subagent fan-out — measured, on its own denominator#

    +

    Not observed on any of the 14 reset runs. Every one carried a main-scope window, which never reads the subagent transcripts, so a zero here means not looked at, not none ran. Three runs declared an explore subagent in their own emit; the declared figure is kept beside the measured one and never merged with it.

    +

    7. What is missing#

    + + + + + + + + + diff --git a/docs/metrics/METRICS.md b/docs/metrics/METRICS.md new file mode 100644 index 0000000..641653c --- /dev/null +++ b/docs/metrics/METRICS.md @@ -0,0 +1,157 @@ +# TechieFlow — Development Metrics + + + +**Snapshot as of 2026-09-07** · project_type `framework` · schema v1 + +This repository is the framework itself, not an application. It **does** have a checklist of its own — `docs/TechieFlow-Requirements.md`, 63 numbered lines each with a stated way to prove it — and since 2026-09-07 it is graded against them by `tests/requirements/run.sh` like any other project. What it has no data for is a *screen* verdict: it has no application to boot, so the gate catch distribution and escape rate below stay empty for a reason that is about screens, not about requirements. Alongside that it measures **the cost of maintaining the framework**, which in this snapshot is almost entirely the seven-session reset of 2026-09-04 to 2026-09-07. + +| Stream | Records | Span | +|---|---|---| +| `runs.jsonl` | 42 | 2026-08-28 → 2026-09-07 | +| `gates.jsonl` | 12 | 2026-09-07 (the framework's own requirement lines, graded for the first time) | +| `sessions.jsonl` | 29 | 2026-08-20 → 2026-09-07 | +| `commits.jsonl` | 52 | 2026-06-25 → 2026-09-07 | +| `misses.jsonl` | 121 miss + 88 miss-fix + 53 miss-amend | 2026-08-28 → 2026-09-07 | + +--- + +## 1. First-pass rate — the framework's own requirements + +**100% of the 12 lines that can be graded, 12 of 63.** Every one passed on its first recorded verdict. + +That number needs its denominator and its date, or it flatters the framework: + +| | | +|---|---| +| Requirement lines in `docs/TechieFlow-Requirements.md` | 63 | +| Graded by a check that runs today | 12 | +| Passed | 12 | +| Failed | 0 | +| Ungraded | 51 | + +**Why only 12.** A line is graded only when its own Check column names something runnable — a self-test, a script, an npm script — and that artefact is run for the verdict. Of the other 51: **14 name a fixture run** (a real command on a real project, which no automated pass can stand in for), **5 name a review** by a person, **1 needs a normal filesystem** (`FR-63`, which a Windows mount cannot grade honestly), **3 are script candidates that were never written**, and **28 describe a script in prose that has no runnable artefact behind it**. Those 28 are the finding: the framework wrote down how it would check itself and then did not build the check. + +**What this figure is not.** These 12 lines were graded for the first time on 2026-09-07, at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: **121 misses logged during the same period** (§5). Read the two together or neither. + +Until 2026-09-07 this section read "no data", on the reasoning that the framework had no checklist to verify. That was wrong: it has had 63 requirement lines since Session 2, named in every session restart prompt. Logged as `MISS-TechieFlow-20260907-16`. + +## 2. Gate catch distribution + +**No data**, and here the original reasoning does hold: a gate distribution answers "which of the seven checks caught the failure", and those checks — build, acceptance, data, visual, assets, speed, standards — are applied to a running application's screens. This repository has none. All 12 framework verdicts passed, so there is no failure to attribute in any case. + +Nothing is inferred from the miss stream to fill the gap: the two are computed from different records by different definitions, and presenting one as the other would make the word meaningless. + +## 3. Escape rate + +**No data** from `gates.jsonl`: an escape is a defect that got past every gate to a person, and with no screens to gate there is nothing for one to escape. The miss stream's own "found by a human" share — **41%** — is reported in §5 **beside** this, never merged into it. For the framework that share is the meaningful number, and it says the owner found two of every five framework defects. + +## 4. Throughput and rework — poolable + +| Figure | Value | Note | +|---|---|---| +| Runs recorded | 42 | `framework-reset` 14 · `log-miss` 22 · `fix-issues` 6 | +| Rework ratio | insufficient data | no `build-phase` runs in this repository | +| Batch size | insufficient data | same reason | +| REQ throughput | 1.83 REQ/hour | median across runs; here a "REQ" is a framework requirement line (FR-nn) touched by a maintenance run | +| Sessions and tokens | 29 sessions · 10,415,290 tokens | 3 duplicate session ids collapsed, normal for OpenCode | +| Commit cadence | 2.17 commits per active day | 52 commits over 24 active days | +| Tokens per Verified | insufficient data | nothing is verified in this repository | + +## 5. Misses — what was missed, who missed it, what the fix cost + +121 misses logged: **33 open, 87 resolved, 1 will-not-fix**, with 88 fix records. Will-not-fix is a decision, not a backlog item, so it is not counted as open. + +| Cut | Distribution | +|---|---| +| Class | wrong-behaviour 64 (53%) · unspecified-gap 26 (21%) · partial-implementation 17 (14%) · scope-creep 6 (5%) · other 3 · spec-contradiction 3 · missed-requirement 2 | +| Why it was missed (118 of 121 assessed) | insufficient-verify-method 46 (39%) · missing-checklist-item 40 (34%) · instruction-ignored 26 (22%) · ambiguous-acceptance 5 (4%) · other 1 | +| Whose gap (66 of 66 sorted) | weak-check 33 (50%) · unsaid 18 (27%) · ignored 15 (23%) | +| Found by | agent-review 52 · owner 50 · library-feedback 11 · gate 8 | + +- **Design-miss share: 21%** — one miss in five was the specification's fault, not the build's. +- **Found by a human: 41%.** Reported beside the gate-derived escape rate of §3, never merged into it. +- **55 misses predate the `sort` field** (added 2026-09-07) and are outside the "whose gap" percentages. 53 fields have been completed by `miss-amend` records. +- **One escape carries no `why_missed`.** Something got past every gate and nothing recorded why; that is the most valuable record in the stream and it is still incomplete. + +The headline finding of the "whose gap" cut: **half of what the framework got wrong was a check that was too weak, not a rule nobody had written.** That is why the reset's method was to turn prose into scripts rather than to add prose. + +### 5a. Attribution — `linked` records only + +**5 of 121 records (116 excluded as inferred or unknown.)** An excluded record named a phase that no run record backs, so its model is unknown, and a per-model rate computed from guesses is a routing decision made on invented evidence. + +- by origin phase: log-miss 3 · fix-issues 2 +- by origin agent: flow-master 3 · general 2 +- by origin model: gpt-5.6-sol 3 · unknown 1 · claude-opus-5 1 + +At n=5 this supports no per-model conclusion and none is drawn. **A per-model miss rate is observational, not causal:** which model gets the hard work is not random. + +### 5b. Rework cost — measured and apportioned never combine + +| Attribution | Fix records | Tokens out per miss | +|---|---|---| +| `sole` — measured | 4 | 289,563 (n=4 priced) | +| `shared` — apportioned by equal division, **not a measurement** | 63 | 81,140 (n=63) | +| unattributable — no usable token window | 20 | not costed | + +Four further records stored as `none` do have a measured window; the divisor is recomputed at read time from the misses each run actually closed. **No dollar figure exists**: Claude Code carries `cost_usd: null` permanently and is never priced from a rate card. The only measured money in this repository is $0.230819 across one OpenCode `log-miss` run. + +## 6. Effort per phase — time, tokens, model, fan-out + +42 live run records. Token-window coverage: `main` 28 · `none` 7 · `conversation` 3 · `tree` 1 · absent 1. A window is only as good as its scope, and a run whose window could not be computed is excluded from every token figure rather than averaged in as a zero. + +| Phase | Runs | Wall clock | Tokens out | Tokens in | % out | % time | +|---|---|---|---|---|---|---| +| `framework-reset` | 14 | 56h 30m | 7.9M | 60.8k | 95% | 94% | +| `fix-issues` | 6 | 3h 05m | 418.7k | 6.6M | 5% | 5% | +| `log-miss` | 22 | 29m 45s | 30.4k | 2.9M | 0% | 1% | + +`framework-reset` costing more than `log-miss` is a fact about what those phases are, not a finding about either. + +### 6a. The reset, session by session + +The framework's own maintenance, in the order it ran. Time on six of these fourteen records was computed from the record's own two timestamps, because they predate the `duration_s` field; that is arithmetic on recorded facts, and it is flagged here rather than left to assume. + +| Session | Runs | Wall clock | Tokens out | Files written | +|---|---|---|---|---| +| Sessions 1 to 3 (before the `mode` field) | 3 | 10h 56m | 1.7M | 48 | +| Sitting 4a | 1 | 9h 55m | 1.0M | 2 | +| Sitting 4b | 3 | 24h 38m | 2.6M | 104 | +| Sitting 4c | 1 | 8h 21m | 1.4M | 42 | +| Session 5 | 1 | 53m 53s | 989.5k | 24 | +| Session 6 | 1 | 30m 19s | **unmeasured** | 34 | +| Session 7 | 1 | 20m 37s | 100.7k | 4 | +| Session 7, merge fix | 1 | 18m 24s | 39.4k | 5 | +| Codex removal | 1 | 2m 35s | 4.5k | 41 | +| This metrics pass | 1 | 32m 51s | 45.5k | 4 | +| **Total** | **14** | **56h 30m** | **7.9M over 13 records** | **308** | + +Session 6's own record carries no token window and is excluded from every token figure above rather than counted as zero. That gap is itself a recorded defect (`MISS-TechieFlow-20260907-09`): the emitter accepted a run record with no `ended`, so the run could never be costed. It is fixed, and the fix is what makes the other twelve rows complete. + +Reading the shape: **the four Session 4 sittings account for 43 of the 56.5 hours and 5.0M of the 7.9M output tokens.** That was the work of shrinking every task file, and it cost roughly three quarters of the whole reset. The five sessions that followed — the miss protocol, the readability split, the Playbook prompt, the merge fix and the Codex removal — took two hours and five minutes between them. + +### 6b. Which model did the work + +| Model | Tokens out | Runs | Share | +|---|---|---|---| +| claude-fable-5-1 | 7.7M | 9 | 98% | +| claude-opus-5 | 190.1k | 4 | 2% | +| synthetic (no model recorded) | 0 | 2 | 0% | + +Harness: `claude-code` on all 14 reset runs. Model routing across the whole repository was observed as on-tier 0 · drifted 10 · unknown 12 — observed, never enforced. + +### 6c. Subagent fan-out — measured, on its own denominator + +**Not observed on any of the 14 reset runs.** Every one carried a `main`-scope window, which never reads the subagent transcripts, so a zero here means *not looked at*, not *none ran*. Three runs declared an `explore` subagent in their own emit; the declared figure is kept beside the measured one and never merged with it. + +## 7. What is missing + +- **51 of the framework's 63 requirement lines are ungraded** — 28 describe a script nobody built, 14 need a fixture run, 5 need a review, 3 are unbuilt candidates and 1 needs a filesystem this machine cannot provide. That is the largest gap on this page, and it is a gap in the framework's own verification, not in its telemetry. +- **No gate catch distribution or escape rate**, because both describe a running application's screens and this repository has none. +- **One run record has no token window** (Session 6), and one miss has no `why_missed`. Both are named above rather than filled in. +- **55 misses predate the `sort` field** and are outside the "whose gap" percentages. They can be completed one at a time with `tf-emit.sh --amend sort `. +- **Attribution covers 5 of 121 misses.** Until more misses carry a linked run, no per-model or per-phase miss rate can be published from this repository. +- **Sessions 1 to 3 share one row** because the `mode` field arrived with Sitting 4a; their three records are individually intact in the stream. diff --git a/docs/metrics/commits.jsonl b/docs/metrics/commits.jsonl index 85e28a3..2e7b08b 100644 --- a/docs/metrics/commits.jsonl +++ b/docs/metrics/commits.jsonl @@ -50,3 +50,4 @@ {"v":1,"ts":"2026-09-05T07:14:43Z","kind":"commit","app":"TechieFlow","sha":"b3b07a1","files":1,"insertions":1,"deletions":0,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} {"v":1,"ts":"2026-09-05T07:15:22Z","kind":"commit","app":"TechieFlow","sha":"78fa676","files":0,"insertions":0,"deletions":0,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} {"v":1,"ts":"2026-09-07T07:59:23Z","kind":"commit","app":"TechieFlow","sha":"a74672e","files":0,"insertions":0,"deletions":0,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} +{"v":1,"ts":"2026-09-07T09:42:14Z","kind":"commit","app":"TechieFlow","sha":"a255b5b","files":94,"insertions":368,"deletions":3683,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} diff --git a/docs/metrics/gates.jsonl b/docs/metrics/gates.jsonl index e69de29..f4e787a 100644 --- a/docs/metrics/gates.jsonl +++ b/docs/metrics/gates.jsonl @@ -0,0 +1,12 @@ +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-14","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-54","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-24","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/verify/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-26","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-31","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-32","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-40","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-42","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-43","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-44","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-47","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-62","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} diff --git a/docs/metrics/misses.jsonl b/docs/metrics/misses.jsonl index 1f20b1c..b9a1bf5 100644 --- a/docs/metrics/misses.jsonl +++ b/docs/metrics/misses.jsonl @@ -258,3 +258,6 @@ {"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-14","req_id":null,"fix_run_id":"2026-09-07T08:10:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T08:28:24Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":120,"tokens_out":39397,"tokens_cache_read":24174545,"tokens_cache_write":58742,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"shared:2"} {"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260904-14","req_id":null,"fix_run_id":"2026-09-07T08:55:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T08:57:35Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":18,"tokens_out":4476,"tokens_cache_read":4817291,"tokens_cache_write":4746,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"shared:2"} {"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-10","req_id":null,"fix_run_id":"2026-09-07T08:55:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T08:57:35Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":18,"tokens_out":4476,"tokens_cache_read":4817291,"tokens_cache_write":4746,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"shared:2"} +{"kind":"miss","miss_id":"MISS-TechieFlow-20260907-15","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"other","severity":"major","why_missed":"insufficient-verify-method","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T10:00:43Z","failure_class":"other","what":"A run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved.","sort":"weak-check","v":1,"ts":"2026-09-07T10:00:44Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} +{"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-15","req_id":null,"fix_run_id":"2026-09-07T09:30:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T10:02:51Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":132,"tokens_out":45502,"tokens_cache_read":37354395,"tokens_cache_write":81347,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"sole"} +{"kind":"miss","miss_id":"MISS-TechieFlow-20260907-16","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"other","severity":"blocker","why_missed":"instruction-ignored","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T10:44:42Z","failure_class":"other","what":"The framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all.","sort":"ignored","v":1,"ts":"2026-09-07T10:44:42Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} diff --git a/docs/metrics/runs.jsonl b/docs/metrics/runs.jsonl index 4c9f014..3ba5918 100644 --- a/docs/metrics/runs.jsonl +++ b/docs/metrics/runs.jsonl @@ -38,3 +38,6 @@ {"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T08:27:58Z","ended":"2026-09-07T08:27:58Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T08:27:59Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":1240},"tokens_in":2,"tokens_out":1240,"tokens_cache_read":414873,"tokens_cache_write":516,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} {"kind":"run","app":"TechieFlow","cmd":"framework-reset","mode":"session-7-merge-fix","started":"2026-09-07T08:10:00Z","reqs_touched":["FR-51","FR-63"],"reqs_count":2,"subagents":[],"files_written":5,"build_result":"pass","yolo":false,"v":1,"ts":"2026-09-07T08:28:24Z","ended":"2026-09-07T08:28:24Z","duration_s":1104,"project_type":"framework","harness":"claude-code","attempt":1,"model":"claude-opus-5","model_tokens_out":{"claude-opus-5":39397},"tokens_in":120,"tokens_out":39397,"tokens_cache_read":24174545,"tokens_cache_write":58742,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0} {"kind":"run","app":"TechieFlow","cmd":"framework-reset","mode":"codex-removal","started":"2026-09-07T08:55:00Z","reqs_touched":["FR-42","FR-62"],"reqs_count":2,"subagents":[],"files_written":41,"build_result":"pass","yolo":false,"v":1,"ts":"2026-09-07T08:57:35Z","ended":"2026-09-07T08:57:35Z","duration_s":155,"project_type":"framework","harness":"claude-code","attempt":2,"model":"claude-opus-5","model_tokens_out":{"claude-opus-5":4476},"tokens_in":18,"tokens_out":4476,"tokens_cache_read":4817291,"tokens_cache_write":4746,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0} +{"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T10:00:43Z","ended":"2026-09-07T10:00:43Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T10:00:44Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":776},"tokens_in":2,"tokens_out":776,"tokens_cache_read":583432,"tokens_cache_write":4667,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} +{"kind":"run","app":"TechieFlow","cmd":"framework-reset","mode":"metrics","started":"2026-09-07T09:30:00Z","reqs_touched":["FR-38"],"reqs_count":1,"subagents":[],"files_written":4,"build_result":"pass","yolo":false,"v":1,"ts":"2026-09-07T10:02:51Z","ended":"2026-09-07T10:02:51Z","duration_s":1971,"project_type":"framework","harness":"claude-code","attempt":1,"model":"claude-opus-5","model_tokens_out":{"claude-opus-5":45502},"tokens_in":132,"tokens_out":45502,"tokens_cache_read":37354395,"tokens_cache_write":81347,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0} +{"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T10:44:42Z","ended":"2026-09-07T10:44:42Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T10:44:42Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":268},"tokens_in":2,"tokens_out":268,"tokens_cache_read":635040,"tokens_cache_write":627,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} diff --git a/tests/mirror/run.sh b/tests/mirror/run.sh index c378531..68f5a35 100644 --- a/tests/mirror/run.sh +++ b/tests/mirror/run.sh @@ -63,6 +63,11 @@ for f in "$ROOT/.tfcore/tasks"/*.md; do w=$(wc -w < "$f"); [[ $w -le 7000 ]] || { bad "$(basename "$f") is $w words, over the 7,000-word frontier budget (FR-43)"; over=$((over+1)); } done [[ $over -eq 0 ]] && ok "every task file is under 7,000 words (FR-43)" +# FR-26: the verify task specifically, which was 11,850 words and caused 63 of 128 recorded +# misses. Its own cap is 4,000, tighter than the 7,000 every task shares. +vw=$(wc -w < "$ROOT/.tfcore/tasks/verify-phase.md" 2>/dev/null || echo 99999) +[[ $vw -le 4000 ]] && ok "verify-phase.md is $vw words (FR-26: at most 4,000)" \ + || bad "verify-phase.md is $vw words, over the 4,000-word cap (FR-26)" shared=$(cat "$ROOT/.tfcore/tasks"/_*.md | wc -w) [[ $shared -le 3000 ]] && ok "shared rule files total $shared words (FR-44: under 3,000)" || bad "shared rule files total $shared words, over 3,000 (FR-44)" overp=0 diff --git a/tests/requirements/run.sh b/tests/requirements/run.sh new file mode 100644 index 0000000..7eccaa7 --- /dev/null +++ b/tests/requirements/run.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# tests/requirements/run.sh — grade the framework against its OWN checklist. +# +# `docs/TechieFlow-Requirements.md` is the framework's checklist: 63 lines, each with a stated +# way to prove it. Until 2026-09-07 nothing ever ran it, so the framework demanded of every +# application a verification it never performed on itself, and its metrics reported "no data" +# for first-pass rate as though it had no requirements at all. It has 63. +# +# WHAT THIS GRADES, and what it refuses to grade. A line is graded ONLY when its own Check +# column names something runnable — a self-test, a utility script, or an npm script. That +# artefact is run, and its exit status is the line's verdict. A line whose check is a fixture +# run (a real command on a real project) or a review (a person reads something) is NOT graded: +# it is reported as ungraded with the reason. Guessing there would be worse than the silence it +# replaces, and the ungraded count is the honest measure of how much of this checklist still +# rests on someone remembering. +# +# bash tests/requirements/run.sh grade and print +# bash tests/requirements/run.sh --emit also append one gate record per graded line +# +# Exit 0 when no graded line failed. +set -u +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)" +EMIT=0; [[ "${1:-}" == "--emit" ]] && EMIT=1 +cd "$ROOT" || exit 2 + +REQ_DOC="docs/TechieFlow-Requirements.md" +[[ -f $REQ_DOC ]] || { echo "no $REQ_DOC"; exit 2; } + +# ---- run each distinct artefact once, cache its exit status ----------------------------- +declare -A ARTEFACT_RC +run_artefact() { + local key="$1" cmd="$2" + if [[ -z "${ARTEFACT_RC[$key]:-}" ]]; then + if bash -c "$cmd" >/dev/null 2>&1; then ARTEFACT_RC[$key]=0; else ARTEFACT_RC[$key]=1; fi + echo " ran ${key} -> $([[ ${ARTEFACT_RC[$key]} -eq 0 ]] && echo pass || echo FAIL)" >&2 + fi + return "${ARTEFACT_RC[$key]}" +} + +# The Check column names these; each maps to the command that runs it. +artefact_cmd() { + case "$1" in + tests/mirror/run.sh) echo "bash tests/mirror/run.sh" ;; + tests/doc-check/run.sh) echo "bash tests/doc-check/run.sh" ;; + tests/bugs/run.sh) echo "bash tests/bugs/run.sh" ;; + tests/verify/run.sh) echo "bash tests/verify/run.sh" ;; + tests/goal/run.sh) echo "bash tests/goal/run.sh" ;; + "npm run test:install") echo "npm run test:install" ;; + *) echo "" ;; + esac +} + +echo "Grading the framework against $REQ_DOC" +echo "" + +graded=0; passed=0; failed=0; ungraded=0 +declare -a EMIT_LINES=() +RUN_ID="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +while IFS=$'\t' read -r fid kind artefact; do + [[ -n "$fid" ]] || continue + # FR-63's own check says to run it on a normal filesystem: on a Windows mount every file + # reports mode 777, so the installer marks a file executable where the shell route does not + # and the comparison shows one false difference. Grading it here would record a failure the + # code does not have, so it is reported ungraded with the reason instead. + if [[ "$artefact" == "npm run test:install" && "$ROOT" == /mnt/* ]]; then + ungraded=$((ungraded+1)) + printf '%-7s %-11s %-22s %s\n' "$fid" "ungraded" "-" "needs a normal filesystem, not a Windows mount" + continue + fi + if [[ -n "$artefact" ]]; then + cmd="$(artefact_cmd "$artefact")" + if [[ -n "$cmd" ]]; then + graded=$((graded+1)) + if run_artefact "$artefact" "$cmd"; then + verdict="Verified"; gate="null"; passed=$((passed+1)) + else + verdict="FAIL"; gate='"acceptance"'; failed=$((failed+1)) + fi + printf '%-7s %-11s %-22s %s\n' "$fid" "$verdict" "$artefact" "$kind" + EMIT_LINES+=("{\"kind\":\"gate\",\"run_id\":\"$RUN_ID\",\"req_id\":\"$fid\",\"req_class\":\"FR\",\"verdict\":\"$verdict\",\"gate\":$gate,\"gates_run\":[\"acceptance\"],\"proof\":\"$artefact\"}") + continue + fi + fi + ungraded=$((ungraded+1)) + printf '%-7s %-11s %-22s %s\n' "$fid" "ungraded" "-" "$kind" +done < <(python3 - "$REQ_DOC" <<'PY' +import re, sys +for line in open(sys.argv[1]): + m = re.match(r'^\|\s*(FR-\d+)\s*\|(.*)$', line) + if not m: continue + cells = [c.strip() for c in m.group(2).split('|')] + check = cells[1] if len(cells) > 1 else "" + low = check.lower() + if "script candidate" in low: kind = "script candidate, not built" + elif low.startswith("fixture run"): kind = "needs a fixture run" + elif low.startswith("review"): kind = "needs a review" + elif low.startswith("script"): kind = "script" + else: kind = "other" + art = "" + for cand in ("tests/mirror/run.sh", "tests/doc-check/run.sh", "tests/bugs/run.sh", + "tests/verify/run.sh", "tests/goal/run.sh", "npm run test:install"): + if cand in check: + art = cand; break + if not art and kind == "script": + kind = "script described, no runnable artefact named" + print(f"{m.group(1)}\t{kind}\t{art}") +PY +) + +total=$((graded + ungraded)) +echo "" +echo "framework requirements: $total lines · graded $graded (passed $passed, failed $failed) · ungraded $ungraded" +echo " ungraded means the line's own check is a fixture run, a review, or a script that was" +echo " described but never written. It is not a pass and it is not a failure." + +if [[ $EMIT -eq 1 && ${#EMIT_LINES[@]} -gt 0 ]]; then + printf '%s\n' "${EMIT_LINES[@]}" | bash .tfcore/utils/tf-emit.sh gates + echo "emitted ${#EMIT_LINES[@]} gate record(s) under run_id $RUN_ID" +fi + +[[ $failed -eq 0 ]] From ddd075a32cde68279ba38d0d897560b50ef166b9 Mon Sep 17 00:00:00 2001 From: S Ravi Kumar Date: Mon, 7 Sep 2026 17:22:37 +0530 Subject: [PATCH 2/2] Updated Metrics --- .tfcore/telemetry/SCHEMA.md | 2 +- docs/AI-First-Playbook-Review-Prompt.md | 190 ++++++----- docs/CHANGELOG.html | 15 + docs/CHANGELOG.md | 16 + docs/TechieFlow-Misses.html | 15 +- docs/TechieFlow-Misses.md | 11 +- docs/TechieFlow-Requirements.md | 34 +- docs/TfLens-Metrics-Update-Prompt.html | 412 ++++++++++++++++++++++++ docs/TfLens-Metrics-Update-Prompt.md | 121 +++++++ docs/metrics/METRICS.html | 27 +- docs/metrics/METRICS.md | 27 +- docs/metrics/commits.jsonl | 1 + docs/metrics/gates.jsonl | 29 ++ docs/metrics/misses.jsonl | 5 + docs/metrics/runs.jsonl | 4 + docs/metrics/sessions.jsonl | 1 + tests/requirements/checks.sh | 215 +++++++++++++ tests/requirements/run.sh | 18 ++ 18 files changed, 1010 insertions(+), 133 deletions(-) create mode 100644 docs/TfLens-Metrics-Update-Prompt.html create mode 100644 docs/TfLens-Metrics-Update-Prompt.md create mode 100644 tests/requirements/checks.sh diff --git a/.tfcore/telemetry/SCHEMA.md b/.tfcore/telemetry/SCHEMA.md index 6126759..2c62ab7 100644 --- a/.tfcore/telemetry/SCHEMA.md +++ b/.tfcore/telemetry/SCHEMA.md @@ -93,7 +93,7 @@ So `tf-emit.sh` detects it and injects it. **Never write `harness` into an emit | Field | Type | Values / notes | |---|---|---| -| `cmd` | string | `day1-brownfield` \| `day1-greenfield` \| `split-brd` \| `mockups` \| `build-phase` \| `verify-phase` \| `fix-issues` \| `triage-issues` \| `log-miss` \| `devguide` \| `productguide` \| `handoff-phase` \| `refresh-status` \| `amend-docs` \| `deploy-checklist` (added 2026-09-06) | +| `cmd` | string | `day1-brownfield` \| `day1-greenfield` \| `split-brd` \| `mockups` \| `build-phase` \| `verify-phase` \| `fix-issues` \| `triage-issues` \| `log-miss` \| `devguide` \| `productguide` \| `handoff-phase` \| `refresh-status` \| `amend-docs` \| `deploy-checklist` (added 2026-09-06) \| `metrics-report` \| `generate-html` \| `render-workflow-docs` \| `triage-and-fix` \| `framework-reset` (framework maintenance, D-19; all five added 2026-09-07, after FR-37's check was built and found 27 records already carrying a value the schema did not list). A value outside this list is a defect in the emitting task, not in the reader: the reader keeps it. | | `mode` | string \| null | `build` \| `fix`. `build-phase` already distinguishes these (FIX mode) — capture it; the ratio is the rework metric. `null` for commands with no mode. | | `started` | string | ISO-8601 UTC. When the task began — the timestamp you noted at step 0, not "now minus a guess". | | `ended` | string | ISO-8601 UTC. Normally equal to `ts`. | diff --git a/docs/AI-First-Playbook-Review-Prompt.md b/docs/AI-First-Playbook-Review-Prompt.md index a11ae71..fefbbce 100644 --- a/docs/AI-First-Playbook-Review-Prompt.md +++ b/docs/AI-First-Playbook-Review-Prompt.md @@ -1,25 +1,25 @@ -# AI-First Playbook — Review Session Prompt (version 2) +# AI-First Playbook — Review Session Prompt (version 3) | | | |---|---| | Purpose | The text the owner pastes into a fresh Claude Code window to review the AI-First Playbook, the team edition, and produce a plan for fixing it. | | Audience | The owner, and the review session that reads it. | -| Status | **Version 2, written 2026-09-07 at the close of the TechieFlow reset (Session 7).** It replaces version 1, which was drafted before those sessions and is kept only in git history. Ready to run. | -| Companion | `TechieFlow-Reset-Plan-2026-09-04.md`, `TechieFlow-Document-Schemas.md`, `TechieFlow-Requirements.md`, `TechieFlow-How-It-Works.md`, `docs/CHANGELOG.md` | +| Status | **Version 3, written 2026-09-07 after the TechieFlow reset closed and its own metrics were built.** Version 2 was written earlier the same day, before the framework was graded against its own checklist; that grading changed what this prompt asks for. Versions 1 and 2 are in git history. Ready to run. | +| Companion | `TechieFlow-Reset-Plan-2026-09-04.md`, `TechieFlow-Requirements.md`, `TechieFlow-Document-Schemas.md`, `docs/CHANGELOG.md`, `docs/metrics/METRICS.md` | --- -## 1. What changed between version 1 and version 2 +## 1. What version 3 adds -Version 1 assumed the Playbook had TechieFlow's disease and told the review to look for the same pattern. A measurement on 2026-09-07 says the shape is different, and the difference decides where the work goes. +Version 2 already carried the reset's methods. Then the framework was measured against its own requirements for the first time, and that produced the finding this version is built around. -**The prose the reader sees is lean.** The ten phase files total 3,572 words, an average of 357 each. Shouted rules of the MUST / NEVER / BANNED kind number 61 across the whole repository, against 622 in TechieFlow before its reset, and 46 of those 61 sit in one folder. Fifteen scripts already exist. +**TechieFlow had 63 requirement lines and had never graded one.** They were written in Session 2, named in every session restart prompt, and each carried a stated check. Nothing ran them. The framework's own metrics reported "no first-pass rate" on the reasoning that it had no checklist to verify — while the checklist sat in `docs/`. When a grader was finally built it took an afternoon and immediately found three real defects, including two that had been live for days. -**The prose the agent reads is not.** The shipped OpenCode harness carries a **verifier agent of 8,630 words** and fifteen command files totalling **29,800**, the largest of them 4,855. TechieFlow's caps after its reset are 1,500 words for a persona and 7,000 for a command, and its own evidence is that its 11,850-word verify task was the origin of 63 of 128 recorded misses. The Playbook's verify path is the same shape as the file that hurt TechieFlow most. +Worse than the not-running was **what the grading revealed about the checks themselves**: of 63 lines, 42 said "script", and **28 of those described a script in prose that nobody had ever written**. A requirement that names a check it does not have reads exactly like a requirement that is enforced. That is the single most expensive habit the reset found, and it is the thing this review must look for in the Playbook first. -**And there is a third thing neither version predicted.** `verification/` holds **174 files and 196,498 words** of committed run evidence: three dated campaigns, each keeping a complete copy of an installed target. That is 62 percent of the repository's 317,385 markdown words, it is not shipped by the npm package, and it is why the same 8,630-word verifier file exists six times in the tree. TechieFlow bans exactly this: run material lives under `tests/.artifacts/` and is swept after seven days, because a repository that keeps every run's output makes every later search return stale copies. +So version 3 changes the ask. Version 2 said "write a requirements list for the Playbook, about 30 lines, each with a way to check it". That is not enough, and it is how TechieFlow got 28 phantom checks. Version 3 says: **write fewer lines, build the grader in the same session, and report the ungraded count as the headline.** A checklist nobody can run is documentation pretending to be enforcement. -So version 2 does three things version 1 did not. It hands the review a measured starting point to confirm or refute rather than a borrowed diagnosis. It carries the methods the TechieFlow sessions proved. And it names what went wrong in those sessions, including the two ways these very numbers were nearly reported wrong, so this review does not repeat it. +Everything version 2 measured about the Playbook still stands and is repeated in Step 1 for confirmation. **How the work is split, unchanged.** The Playbook is edited in Claude Code, because that is where the owner works fastest. It is proven only in OpenCode, because that is the harness its users have. A change that works in Claude Code and fails in OpenCode is not done. @@ -36,9 +36,9 @@ OpenCode. Corporate teams are the audience; they avoid single-vendor tools, so O primary harness and must stay so. Claude Code is secondary here. The Playbook and its solo sibling TechieFlow were both borrowed from BMAD and then corrected -incident by incident over six months. TechieFlow has just been through a seven-session reset. What -that reset learned is written into the method below. Do not assume the two frameworks have the same -faults: measure this one. +incident by incident over six months. TechieFlow has just been through a seven-session reset and +has been graded against its own requirements for the first time. What that cost is written into the +method below. Do not assume the two frameworks have the same faults: measure this one. RULES FOR THIS SESSION - Plain English in everything you write. Short sentences. Any term of art gets a one-clause @@ -50,7 +50,10 @@ RULES FOR THIS SESSION contradicts something stated in this prompt, the number wins and you say so. - Keep OpenCode first-class in every proposal. Never propose removing OpenCode support, the operating contract, or the npm packaging. -- A proposal that cannot be checked by a script is a weak proposal. Say so when you make one. +- NEVER propose a check you are not also prepared to build. If a line needs a check that cannot be + written this session, say "review" and mean it. A requirement that names a script nobody wrote is + worse than one that admits it rests on a person: it reads as enforced and is not. TechieFlow + carried 28 of those. STEP 1 — MEASURE THE SURFACE Two counting rules, both paid for. Break either and every number below is wrong. @@ -60,11 +63,10 @@ Two counting rules, both paid for. Break either and every number below is wrong. be a number reads as "nothing here" instead of "I did not look". That mistake was made while preparing this prompt and hid 174 files and 196,498 words. - Count HIDDEN paths. Use `find`, not a shell glob, and not a search tool that honours .gitignore. - A harness lives in a dot-directory: `.opencode/`. When this prompt's own Step 1 was run through - OpenCode on 2026-09-07 it reported verification/ as 120 files and 82,168 words, against the true - 174 and 196,498, because `**/*.md` skips dot-directories. It under-reported by 58 percent and - looked entirely plausible. This is the same defect TechieFlow recorded as D-21: the framework's - own tree is invisible to ordinary search, so an agent concludes a file is not there when it is. + A harness lives in a dot-directory: `.opencode/`. When Step 1 was run through OpenCode on + 2026-09-07 it reported verification/ as 120 files and 82,168 words against the true 174 and + 196,498, because `**/*.md` skips dot-directories. It under-reported by 58 percent and looked + entirely plausible. Confirm or refute this snapshot, taken 2026-09-07. Print your own figures beside it. @@ -84,50 +86,58 @@ Confirm or refute this snapshot, taken 2026-09-07. Print your own figures beside 1. Word count of every markdown file, grouped by folder, sorted by size, with file counts. 2. For each of the ten phases, what an OpenCode agent actually loads when that phase runs. Trace it from opencode.json, AGENTS.md and the harness folder: which files, in what order, total words. - That total is the instruction surface for the phase, and it is the number that matters, not the - phase document's own size. Say plainly which phase reads the most before its first useful step. + That total is the instruction surface for the phase, not the phase document's own size. Say + plainly which phase reads the most before its first useful step. 3. Count prose rules and script-enforced rules side by side, and say where each cluster sits. -4. List every document in docs/ with its audience (team lead, developer, agent, owner) and whether - anything in the repo points at it. Flag every document nothing points at. Flag separately any - document that belongs to the other framework rather than this one. -5. Report every file that exists more than once in the tree with the same name and near-identical - content. Duplication is why an edit lands in one copy and not the other. +4. List every document in docs/ with its audience and whether anything in the repo points at it. + Flag every document nothing points at, and separately any that belongs to the other framework. +5. Report every file that exists more than once with the same name and near-identical content. -STEP 2 — THE MISS AND TELEMETRY DATA +STEP 2 — THE EXISTING CHECKS: WHICH ARE REAL +For every rule the Playbook states as enforced — in AGENTS.md, the phase files, the scripts, the +release workflow — answer one question: is there an artefact that fails when the rule is broken, +and can you run it right now? Print a table: + + | The rule | Where it is stated | The check it claims | Runs today? | Proof | + +"Runs today" is yes only if you ran it in this session and saw it pass or fail. Anything else is no. +Count the yes and the no. That ratio is the Playbook's real enforcement, and it is the number the +owner should see first — TechieFlow's equivalent was 12 of 63 before this work and 29 of 63 after. + +STEP 3 — THE MISS AND TELEMETRY DATA 1. If a misses stream or docs/metrics exists, summarise misses by cause, by phase, and by who found them. If none exists, say so plainly: a framework that cannot show what it got wrong cannot show it improved, and that is the single finding most worth fixing first. -2. Read Ai-First-Playbook-Gap.md and docs/Decisions.md. List the rules that were added in reaction - to an incident, and mark each prose or enforced. +2. Read Ai-First-Playbook-Gap.md and docs/Decisions.md. List the rules added in reaction to an + incident, and mark each prose or enforced. -STEP 3 — EVERY PROSE FILE, ONE TABLE +STEP 4 — EVERY PROSE FILE, ONE TABLE Start with what the agent reads, not what the reader reads: harness/opencode/agent/verifier.md (8,630 words) and the four largest command files, then the rest of harness/, then phases/ and templates/. Shrinking the verifier shrinks every verify run, and the equivalent file was the single -worst source of misses in TechieFlow. For each file, print one row per block of the file: +worst source of misses in TechieFlow. For each file, print one row per block: | # | What the block says, in one line | Verdict | Why | Verdict is exactly one of: KEEP AS WORDS (it needs judgement), SCRIPT (it is mechanical, and name -the script), DELETE (it duplicates another file or states the obvious). This is the format the -TechieFlow reset used on every task file; it worked because the owner could rule on a row without -reading the block, and could always ask to see the block. +the script), DELETE (it duplicates another file or states the obvious). The owner can rule on a row +without reading the block, and can always ask to see the block. -STEP 4 — THE TEMPLATES BECOME SCHEMAS +STEP 5 — THE TEMPLATES BECOME SCHEMAS A template that only advises produces a document that drifts. Propose for each template a schema -block: the required sections in order, the optional ones, a word budget per project size as a -TARGET and a MAXIMUM, and the rules every row must follow. A single hard number makes the agent -truncate, which is why the budget is always a pair. +block: required sections in order, the optional ones, a word budget per project size as a TARGET +and a MAXIMUM, and the rules every row must follow. A single hard number makes the agent truncate, +which is why the budget is always a pair. -Give one worked example in full, for templates/checklist-item-template.md, and include the rule +Give one worked example in full, for templates/checklist-item-template.md, including the rule TechieFlow found mattered most: every acceptance line reads "When on , then " -at most 30 words, target 20, holding one behaviour. Fourteen recorded misses across the owner's -projects were traced to acceptance lines that allowed two honest readings. +at most 30 words, target 20, holding one behaviour. Fourteen recorded misses were traced to +acceptance lines that allowed two honest readings. -STEP 5 — THE MISS PROTOCOL +STEP 6 — THE MISS PROTOCOL Propose it as four questions asked in order, each with one fixed response, and a field on the record that stores which one it was: @@ -136,68 +146,79 @@ that stores which one it was: 3. Was there a check, and did it fail to catch it? yes -> fix the check, not the prose. 4. Was it written and ignored anyway? yes -> make it a script or a gate, or delete it. -The fourth response is the important one: a rule ignored twice never gets a third paragraph. -TechieFlow's own numbers say why the third question earns its place — of 61 sorted misses, -31 (51%) were "the check was too weak", 16 (26%) "nobody said it", 14 (23%) "said and ignored". -Half of a framework's failures are its checks, not its words. +TechieFlow's own numbers say why the third question earns its place: of its sorted misses, about +half were "the check was too weak", a quarter "nobody said it", a quarter "said and ignored". Half +of a framework's failures are its checks, not its words. The fourth response is the one with teeth: +a rule ignored twice never gets a third paragraph. -STEP 6 — THE INSTRUCTION BUDGET +STEP 7 — THE INSTRUCTION BUDGET Propose a budget per phase, expressed per model tier rather than as one number, and say how a phase -document is to be written so a small budget can be met without losing steps: a short core, plus -reference sections loaded only when a step needs them. State each phase's current figure from -Step 1.2 against the budget you propose. +document is written so a small budget can be met without losing steps: a short core, plus reference +sections loaded only when a step needs them. State each phase's current figure against the budget. -STEP 7 — THE PLAN +STEP 8 — THE PLAN Write docs/Playbook-Reset-Plan.md with these sections, in this order: 1. What is genuinely right and should be kept. Name files. 2. What grew without earning its place. Name files, sizes, and why. 3. The instruction surface per phase today, and a target for each. - 4. The keep / script / delete table from Step 3, consolidated. + 4. The keep / script / delete table from Step 4, consolidated. 5. Which documents to merge, shrink, or move out of the reader's path. A corporate adopter needs a getting-started, the ten phases, the templates, and one operating guide. Everything else must justify itself. Say where the rest goes; nothing is deleted without a home. 5b. What to do about verification/, which is 62 percent of the repository. Say what the campaigns prove, whether anything still depends on them, what evidence a future run should keep and for - how long, and where it should live so it is not committed. TechieFlow's answer was a single - ignored folder swept after seven days, with the durable proof kept as a self-test that anyone - can re-run instead of as a stored copy of its output. Recommend, do not assume: these campaigns - may be the only record that the OpenCode-only install was ever proven. - 6. A requirements list for the Playbook itself: about 30 lines of "the Playbook must …", each with - a way to check it against a fixture repository under OpenCode. Mark each check script, fixture - run, or review. Every review check is a candidate to become a script, and say so. - 7. The miss protocol from Step 5, as it will be recorded and reported. - 8. An ordered list of work sessions to carry the plan out, each with a goal, the inputs the owner - brings, and the output file. Put the shared files that every phase loads first: shrinking those - shrinks every phase at once. Say so in the plan, because it is the one step that is out of - life-cycle order. + how long, and where it should live so it is not committed. TechieFlow's answer was one ignored + folder swept after seven days, with the durable proof kept as a self-test anyone can re-run + instead of a stored copy of its output. Recommend, do not assume: these campaigns may be the + only record that the OpenCode-only install was ever proven. + 6. A requirements list for the Playbook itself — AND ITS GRADER. This section replaces version 2's + "about 30 lines, each with a way to check it", which is how TechieFlow ended up with 28 checks + that did not exist. The rules: + - Fewer lines, each one testable. Twenty that can be graded beat forty that cannot. + - Every line is marked script, fixture run, or review, and the three are counted separately. + - A line marked "script" names the artefact that runs it, and that artefact is written in the + same session as the line. If it cannot be, the line is marked "review" instead. + - The grader is part of the deliverable: one command that walks the list, runs what can be + run, and reports every other line as ungraded WITH ITS REASON. Never guess a verdict. + - The headline number is not how many pass. It is **how many can be graded at all**. Report it + as "N of M graded", and treat the ungraded count as the backlog it is. + - The grader writes one verdict record per line into the telemetry, so the Playbook's own + compliance is visible beside the projects it builds. + 7. The miss protocol from Step 6, as it will be recorded and reported. + 8. An ordered list of work sessions, each with a goal, the inputs the owner brings, and the output + file. Put the shared files that every phase loads first: shrinking those shrinks every phase at + once. Say so in the plan, because it is the one step out of life-cycle order. Also write docs/Playbook-How-It-Works.md: the ten phases in plain words, one paragraph each, saying what is read, what is written, which template is used, and what OpenCode does around it. A team lead with no AI experience should be able to follow it. This document, not the plan, is what the owner reviews first. -STEP 8 — REPORT -Finish with a short message: the five biggest findings, each one sentence with its number, and the -first session to run. Nothing else. +STEP 9 — REPORT +Finish with a short message: the five biggest findings, each one sentence with its number, the +enforcement ratio from Step 2, and the first session to run. Nothing else. ``` --- ## 3. What went wrong in the TechieFlow sessions -Give this list to the review session if it asks how the method was earned. Each item cost time. - -- **A rule named a check that nobody had written.** The requirement said a script proved it; no script existed, and a private project name sat in the public README for months. When a requirement names a check, the check is written in the same session or the requirement says "review" honestly. -- **A command reported success after its only deliverable was refused.** The miss recorder printed "Miss logged" and an identifier that existed nowhere, because the emitter had rejected a value and appended nothing. A command that reports what it did not do is worse than one that fails loudly. -- **A checklist was frozen and then quietly grown.** Items were added to an owner-reviewed document and the owner was told afterwards. That is how the framework grew unreviewed in the first place. Propose in plain words, get the yes, then edit. -- **Rules were restated instead of enforced.** 622 prose MUST and NEVER statements against 8 hooks. The 8 always held. The 622 held most of the time, and the remainder is where the misses came from. -- **Documents grew until nobody read them.** The briefing reached 344 KB and the README 121 KB, both mostly history. Session 6 cut them to 1,961 and 1,864 words and moved the history to a changelog. Now a script counts them. -- **A single hard budget made the agent truncate.** Every budget became a target and a maximum, and the content rules stop truncation, not the number. -- **The largest file was the worst file.** The verify task at 11,850 words was the origin of 63 of 128 recorded misses. Size is not a cosmetic problem. -- **A self-test check compared the wrong thing** and reported a failure that was not there, wasting time on a fix that already worked. A check is not trusted until it has been made to fail on purpose. -- **Scripts were called done without being run.** The standing rule now is that every script the maintainer touches is run for real, in both harnesses, and its output shown. -- **A measurement taken at the top level of each folder reported zero for folders whose content sits one level down.** It hid 174 files and 196,498 words while preparing this very prompt, and the wrong headline was written into a draft before the recount caught it. Count recursively, and always print the file count beside the word count so a zero that means "I did not look" cannot pass for a zero that means "nothing here". -- **Search tools do not see the framework.** `.tfcore/`, `.claude/` and `.opencode/` are dot-directories and are git-ignored inside a project, so ripgrep, Glob and `git grep` all return nothing for files that are plainly present. TechieFlow recorded it as D-21 after a verify run wrote "not present anywhere in this tree" about a script that existed and closed a gate on it. The rule that followed: confirm a file by reading its literal path, never by searching for its name, and never write "not present" without naming the path tried. It is not a solved problem, only a known one: it recurred on 2026-09-07 while measuring the Playbook, costing 58 percent of one folder's true size. +Give this list to the review session if it asks how the method was earned. Each item cost time, and the last four were found after version 2 of this prompt was written. + +- **A framework never graded itself.** 63 requirement lines, each with a stated check, and not one verdict recorded in the four days they existed. Its own metrics reported "no first-pass rate" because the maintainer reasoned it had no checklist — while the checklist sat in `docs/`, named in every restart prompt. Nobody notices this from inside: it took the owner rejecting a sentence. +- **A rule named a check that nobody had written.** 28 of 63 lines described a script in prose that did not exist. When the grader was built, the first run found technology-specific routing in two tasks that FR-03 forbids, four commands emitting no run record, and five command values the telemetry schema had never listed. All three had been live for days behind a requirement that read as enforced. +- **A number was reported over the wrong denominator.** A phase's total time summed the runs that carried a duration while its tokens summed every run — so the reset showed 16h49m of work against a true 55h57m. Separately, a set of records that all passed reported a first-pass rate of 0%, because the field the rate keys on was absent and absence read as "no". Both were arithmetic the reader could have done from the records it already had. +- **Two delivery routes drifted apart.** The shell scripts and the npm installer produced different projects for four sittings, because nothing said that a change to what a project receives goes into both. A project installed from the package ran without three of its guard hooks. +- **A guard read the whole command line.** The database guard refused a documentation edit and a read-only search because a migration tool's name appeared in the text being written. A rule that blocks the work it was meant to protect gets switched off. +- **A command reported success after its only deliverable was refused.** The miss recorder printed "Miss logged" and an id that existed nowhere, because the emitter had rejected a value and appended nothing. +- **A checklist was frozen and then quietly grown.** Items were added to an owner-reviewed document and the owner was told afterwards. Propose in plain words, get the yes, then edit. +- **Rules were restated instead of enforced.** 622 prose MUST and NEVER statements against 8 hooks. The 8 always held. +- **Documents grew until nobody read them.** The briefing reached 344 KB and the README 121 KB, both mostly history. They are now 2,003 and 1,863 words, and a script counts them. +- **A single hard budget made the agent truncate.** Every budget became a target and a maximum. +- **The largest file was the worst file.** The verify task at 11,850 words was the origin of 63 of 128 recorded misses. It is now 964. +- **A self-test check compared the wrong thing** and reported a failure that was not there. A check is not trusted until it has been made to fail on purpose. +- **Search tools do not see the framework.** Dot-directories are skipped by ripgrep, Glob and shell globs, and git-ignored inside a project. Confirm a file by reading its literal path, never by searching for its name. It recurred on 2026-09-07 while measuring the Playbook, costing 58 percent of one folder's size. +- **Removing a harness is a delivery job, not a deletion job.** When the Codex adapter went, the fix was to make the updater *remove* what it used to deploy. One propagation pass then cleaned 23 projects, repeatably and auditably, instead of anyone deleting folders by hand. --- @@ -206,11 +227,11 @@ Give this list to the review session if it asks how the method was earned. Each The Playbook is not TechieFlow with more people. Four differences change the design. - **One harness, and it enforces differently.** OpenCode has no blocking end-of-turn hook, so a rule that must hold at the end of a turn is a plugin follow-up prompt there, not a hook. Anything proposed as a hook needs its OpenCode form stated beside it, or it is not a rule, it is a hope. -- **A rule that depends on remembering fails faster with more people.** Enforcement belongs in the repository, where a joiner inherits it: scripts, validators, and the pipeline. In a solo framework a habit can substitute for a gate. In a team it cannot. -- **The review gates have named humans.** Phases 02, 06 and 08 are real handoffs between people, where TechieFlow's equivalent is the owner reviewing their own work. Each gate needs its record: what was reviewed, how many corrections came back, and what producing and correcting it cost. Without that the cost of a bad specification stays invisible. -- **Onboarding is a deliverable, not documentation.** A new joiner should reach a working feature in a day, and the getting-started document is the thing that has to make that true. That is the test to apply to `docs/`: if a document does not serve a joiner, an agent, or a gate, it is not on the reading path. +- **A rule that depends on remembering fails faster with more people.** Enforcement belongs in the repository, where a joiner inherits it: scripts, validators, the pipeline. In a solo framework a habit can substitute for a gate. In a team it cannot. +- **The review gates have named humans.** Phases 02, 06 and 08 are real handoffs between people. Each needs its record: what was reviewed, how many corrections came back, and what producing and correcting it cost. TechieFlow added exactly this record kind and it is the only one that prices a specification defect. +- **Onboarding is a deliverable, not documentation.** A new joiner should reach a working feature in a day, and the getting-started document has to make that true. That is the test for `docs/`: if a document does not serve a joiner, an agent, or a gate, it is not on the reading path. -Two further constraints: the Playbook is already distributed on npm, so any change ships through its release checks, and its telemetry runs beside customer work, so records carry identifiers and counts only, never requirement text, prompt text, or anything from a customer's repository. +Two further constraints: the Playbook is already distributed on npm, so any change ships through its release checks — and those checks must run before publish, not after, which is itself a requirement worth grading. Its telemetry runs beside customer work, so records carry identifiers and counts only, never requirement text, prompt text, or anything from a customer's repository. --- @@ -219,4 +240,5 @@ Two further constraints: the Playbook is already distributed on npm, so any chan - The two output documents land in the Playbook's `docs/`. Read the How-It-Works first, mark every line you cannot repeat to a colleague, and have that conversation before any session starts. - The plan's sessions are carried out in Claude Code and proven in OpenCode against a fixture repository. Nothing is marked done from Claude Code alone. - Keep TechieFlow and the Playbook in step on one thing only: the miss protocol and the telemetry schema. Everything else may diverge, because the audiences differ. -- The Playbook's `docs/` currently holds two documents that belong to TechieFlow, `Miss-Telemetry-TechieFlow.md` and `Phase-Efficiency-TfLens-Contract.md`, 7,700 words between them. The review will flag them; deciding where they live is yours. +- The Playbook's `docs/` holds two documents that belong to TechieFlow, `Miss-Telemetry-TechieFlow.md` and `Phase-Efficiency-TfLens-Contract.md`, 7,700 words between them. The review will flag them; deciding where they live is yours. +- **When the plan's Step 8.6 lands, ask one question of it before approving: "how many of these lines can be graded on the day we write them?"** If the answer is most of them, the list is honest. If it is a handful, the list is a wish and the session that wrote it has handed you TechieFlow's 28 phantom checks in a new folder. diff --git a/docs/CHANGELOG.html b/docs/CHANGELOG.html index 06ffed3..d477c2a 100644 --- a/docs/CHANGELOG.html +++ b/docs/CHANGELOG.html @@ -114,6 +114,7 @@

    TechieFlow — Changelog

  • The reset (2026-09-04 to 2026-09-07)
  • The 2026-08-28 review, as it stood at the top of the old briefing
  • Maintenance log (newest first)
    • +
  • 2026-09-07 — the checks built: framework coverage from 12 lines to 29, and three defects it found on the first run
  • 2026-09-07 — the framework graded against its own checklist for the first time
  • 2026-09-07 — the reset measured: docs/metrics/METRICS.md written, and a phase's time was being under-reported
  • 2026-09-07 — the Codex adapter removed, and WORKFLOW.html dropped
    • @@ -215,6 +216,20 @@

    The 20

    Maintenance log (newest first)#

    +

    2026-09-07 — the checks built: framework coverage from 12 lines to 29, and three defects it found on the first run#

    +

    The 28 requirement lines whose Check column described a script nobody had written now have one, or an honest reason they cannot. tests/requirements/checks.sh holds a purpose-built check per line, and the grader runs it: 29 of 63 lines graded, 27 passing, 34 ungraded with the reason stated for each.

    +

    Ten lines gained a real check — technology neutrality, the banned document shapes, the Verified ledger guard, the root-litter guard, the yolo field, the review-record refusals, the schema's command vocabulary, the release order, the honest ended, and the database guard, most of them driving the hook directly and asserting both the refusal and the allowed case. Five more were repointed at the self-test whose planted defect already proved them, and three at the installer test.

    +

    The first run found three defects that had been live for days.

    +
      +
    • FR-03 fails. The framework claims no persona or task names a technology, and the analyst and build-phase hardcode routing to the TrBlazeUI and TechieRag library agents. Left failing, because the fix is the owner's call: either library routing is a stated exception, or the tasks route by requirement prefix to whatever library agents a project has (MISS-TechieFlow-20260907-18). +
      • +
    • FR-34 fails. Four tasks wire no run record: create-doc, generate-html, facilitate-brainstorming-session, create-deep-research-prompt. This is the idea-stage gap D-13 named in Session 1; it is now a number instead of a paragraph. +
      • +
    • The telemetry schema had never listed five command values its own tasks were writing — framework-reset among them, 27 records across the estate. Added (MISS-TechieFlow-20260907-17, fixed). +
      • +
    +

    A fourth, minor, stays open: the database guard reads the whole command line, so it refused a documentation edit and a read-only search because a migration tool's name appeared in the text (MISS-TechieFlow-20260907-19).

    +

    Two documents were written for the work that follows. docs/AI-First-Playbook-Review-Prompt.md is now version 3: it replaces version 2's "about 30 requirement lines, each with a way to check it" with "fewer lines, the grader built in the same session, and the ungraded count as the headline" — because version 2's wording is exactly how this framework acquired 28 checks that did not exist. And docs/TfLens-Metrics-Update-Prompt.md is a hand-over brief for the TfLens team: the five things now in the streams that its pages cannot show, and the three reader-side rules that change numbers it already publishes.

    2026-09-07 — the framework graded against its own checklist for the first time#

    The owner rejected the reasoning in the metrics report, and was right. It said the framework had no first-pass rate because it had "no checklist of its own to verify". It has had one since Session 2: docs/TechieFlow-Requirements.md, 63 numbered lines, each with a stated check, named in every session restart prompt. The framework had been demanding of every application a verification it never performed on itself. Logged as MISS-TechieFlow-20260907-16, severity blocker, sorted ignored — the rule was written down and not followed.

    tests/requirements/run.sh now grades it. A line is graded only when its own Check column names something runnable; that artefact is run and its exit status is the verdict. A line whose check is a fixture run, a review, or a script that was never built is reported ungraded with the reason, never guessed.

    diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b9974d0..f57425f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,22 @@ Everything below predates the reset and is preserved as it was written. ## Maintenance log (newest first) +### 2026-09-07 — the checks built: framework coverage from 12 lines to 29, and three defects it found on the first run + +The 28 requirement lines whose Check column described a script nobody had written now have one, or an honest reason they cannot. `tests/requirements/checks.sh` holds a purpose-built check per line, and the grader runs it: **29 of 63 lines graded, 27 passing, 34 ungraded** with the reason stated for each. + +Ten lines gained a real check — technology neutrality, the banned document shapes, the `Verified` ledger guard, the root-litter guard, the yolo field, the review-record refusals, the schema's command vocabulary, the release order, the honest `ended`, and the database guard, most of them driving the hook directly and asserting both the refusal and the allowed case. Five more were repointed at the self-test whose planted defect already proved them, and three at the installer test. + +**The first run found three defects that had been live for days.** + +- **FR-03 fails.** The framework claims no persona or task names a technology, and the analyst and `build-phase` hardcode routing to the TrBlazeUI and TechieRag library agents. Left failing, because the fix is the owner's call: either library routing is a stated exception, or the tasks route by requirement prefix to whatever library agents a project has (`MISS-TechieFlow-20260907-18`). +- **FR-34 fails.** Four tasks wire no run record: `create-doc`, `generate-html`, `facilitate-brainstorming-session`, `create-deep-research-prompt`. This is the idea-stage gap D-13 named in Session 1; it is now a number instead of a paragraph. +- **The telemetry schema had never listed five command values its own tasks were writing** — `framework-reset` among them, 27 records across the estate. Added (`MISS-TechieFlow-20260907-17`, fixed). + +A fourth, minor, stays open: the database guard reads the whole command line, so it refused a documentation edit and a read-only search because a migration tool's name appeared in the text (`MISS-TechieFlow-20260907-19`). + +**Two documents were written for the work that follows.** `docs/AI-First-Playbook-Review-Prompt.md` is now version 3: it replaces version 2's "about 30 requirement lines, each with a way to check it" with "fewer lines, the grader built in the same session, and the ungraded count as the headline" — because version 2's wording is exactly how this framework acquired 28 checks that did not exist. And `docs/TfLens-Metrics-Update-Prompt.md` is a hand-over brief for the TfLens team: the five things now in the streams that its pages cannot show, and the three reader-side rules that change numbers it already publishes. + ### 2026-09-07 — the framework graded against its own checklist for the first time The owner rejected the reasoning in the metrics report, and was right. It said the framework had no first-pass rate because it had "no checklist of its own to verify". It has had one since Session 2: **`docs/TechieFlow-Requirements.md`, 63 numbered lines, each with a stated check**, named in every session restart prompt. The framework had been demanding of every application a verification it never performed on itself. Logged as `MISS-TechieFlow-20260907-16`, severity blocker, sorted `ignored` — the rule was written down and not followed. diff --git a/docs/TechieFlow-Misses.html b/docs/TechieFlow-Misses.html index 9f1fdf1..ca80031 100644 --- a/docs/TechieFlow-Misses.html +++ b/docs/TechieFlow-Misses.html @@ -111,8 +111,8 @@

    TechieFlow — Misses

    Contents
      -
    1. Open (34)
      2. -
    2. Fixed (87)
      3. +
    3. Open (35)
      4. +
    4. Fixed (89)
    5. Will not fix (1)
    @@ -120,17 +120,18 @@

    TechieFlow — Misses

    AppTechieFlow -Count122 logged: 34 open, 87 fixed, 1 will not fix +Count125 logged: 35 open, 89 fixed, 1 will not fix Sourcedocs/metrics/misses.jsonl, one row per miss record. Rewritten by tf-misses-md.sh on every new record. Never edit it: a wrong row is corrected by a new record. Updated2026-09-07

    Whose gap answers the four questions of the miss protocol: the app's spec did not say it, so the checklist line is fixed; the framework never said it, so one requirement line and a check are added; the check was too weak (a review, or a script that did not fire), so the check is fixed; said and ignored, so the rule becomes a hook or is deleted. not sorted means the record predates the sort or nobody has answered yet; bash .tfcore/utils/tf-emit.sh --amend <miss> sort <spec|unsaid|weak-check|ignored> completes it.

    -

    Open (34)#

    +

    Open (35)#

    - + + @@ -166,10 +167,12 @@

    Open (34)#

    MissFoundWhose gapWhat went wrong
    MISS-TechieFlow-20260907-162026-09-07 by ownersaid and ignoredThe framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all.
    MISS-TechieFlow-20260907-192026-09-07 by ownerthe check was too weakThe database guard reads the whole command line, so it refused a documentation edit and a read-only grep because a migration tool's name appeared in the text being written.
    MISS-TechieFlow-20260907-182026-09-07 by ownerthe check was too weakFR-03 claims no persona or task names a technology, but the analyst and build-phase hardcode routing to the TrBlazeUI and TechieRag library agents, and nothing ever ran the grep that would have said so.
    MISS-TechieFlow-20260907-122026-09-07 by ownersaid and ignoredThe D-21 rule that hidden framework folders are invisible to search was written down and still cost 58 percent of a folder's true size when OpenCode measured the Playbook with a shell glob.
    MISS-TechieFlow-20260907-042026-09-07 by gatethe check was too weakThe readable miss file's unchanged check compared only the header above the Updated line, so an amend that changed a row but no count left the file stale; the bugs self-test caught it before release and the check now compares everything but the date.
    MISS-TechieFlow-20260907-032026-09-07 by agent-reviewthe check was too weakThe cross-project rollup keyed a requirement by its id alone, so REQ-UI-001 of TfLens and REQ-UI-001 of TechieBlog counted as one requirement and the combined first-pass rate printed 72% where the true figure is 48%; found by re-reading the numbers before the explainer, fixed by keying on project and id.
    (no id, record 55)2026-09-05 by ownernot sortedThe first Session 4a task table put the owner questions inside table cells and the rows were too wide to read in a terminal, against the plain-words rule.
    -

    Fixed (87)#

    +

    Fixed (89)#

    + + diff --git a/docs/TechieFlow-Misses.md b/docs/TechieFlow-Misses.md index 05f26f3..28f7860 100644 --- a/docs/TechieFlow-Misses.md +++ b/docs/TechieFlow-Misses.md @@ -3,17 +3,18 @@ | | | |---|---| | App | TechieFlow | -| Count | 122 logged: 34 open, 87 fixed, 1 will not fix | +| Count | 125 logged: 35 open, 89 fixed, 1 will not fix | | Source | `docs/metrics/misses.jsonl`, one row per miss record. Rewritten by `tf-misses-md.sh` on every new record. Never edit it: a wrong row is corrected by a new record. | | Updated | 2026-09-07 | **Whose gap** answers the four questions of the miss protocol: **the app's spec** did not say it, so the checklist line is fixed; **the framework never said it**, so one requirement line and a check are added; **the check was too weak** (a review, or a script that did not fire), so the check is fixed; **said and ignored**, so the rule becomes a hook or is deleted. **not sorted** means the record predates the sort or nobody has answered yet; `bash .tfcore/utils/tf-emit.sh --amend sort ` completes it. -## Open (34) +## Open (35) | Miss | Found | Whose gap | What went wrong | |---|---|---|---| -| MISS-TechieFlow-20260907-16 | 2026-09-07 by owner | said and ignored | The framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all. | +| MISS-TechieFlow-20260907-19 | 2026-09-07 by owner | the check was too weak | The database guard reads the whole command line, so it refused a documentation edit and a read-only grep because a migration tool's name appeared in the text being written. | +| MISS-TechieFlow-20260907-18 | 2026-09-07 by owner | the check was too weak | FR-03 claims no persona or task names a technology, but the analyst and build-phase hardcode routing to the TrBlazeUI and TechieRag library agents, and nothing ever ran the grep that would have said so. | | MISS-TechieFlow-20260907-12 | 2026-09-07 by owner | said and ignored | The D-21 rule that hidden framework folders are invisible to search was written down and still cost 58 percent of a folder's true size when OpenCode measured the Playbook with a shell glob. | | MISS-TechieFlow-20260907-04 | 2026-09-07 by gate | the check was too weak | The readable miss file's unchanged check compared only the header above the Updated line, so an amend that changed a row but no count left the file stale; the bugs self-test caught it before release and the check now compares everything but the date. | | MISS-TechieFlow-20260907-03 | 2026-09-07 by agent-review | the check was too weak | The cross-project rollup keyed a requirement by its id alone, so REQ-UI-001 of TfLens and REQ-UI-001 of TechieBlog counted as one requirement and the combined first-pass rate printed 72% where the true figure is 48%; found by re-reading the numbers before the explainer, fixed by keying on project and id. | @@ -48,10 +49,12 @@ | MISS-TechieFlow-20260904-01 | 2026-09-04 by owner | not sorted | no sentence recorded (wrong-behaviour, other, why: instruction-ignored) | | (no id, record 55) | 2026-09-05 by owner | not sorted | The first Session 4a task table put the owner questions inside table cells and the rows were too wide to read in a terminal, against the plain-words rule. | -## Fixed (87) +## Fixed (89) | Miss | Found | Closed | Whose gap | What went wrong | |---|---|---|---|---| +| MISS-TechieFlow-20260907-17 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the check was too weak | The telemetry schema never gained the command values its own tasks were writing, so 27 run records across the estate carried a cmd the schema does not list and nothing noticed for four days. | +| MISS-TechieFlow-20260907-16 | 2026-09-07 by owner | 2026-09-07 by fix-issues | said and ignored | The framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all. | | MISS-TechieFlow-20260907-15 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the check was too weak | A run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved. | | MISS-TechieFlow-20260907-14 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the framework never said it | The npm installer's framework subfolder list left out standards, so a project migrated from the old layout came out with no coding standards file. | | MISS-TechieFlow-20260907-13 | 2026-09-07 by owner | 2026-09-07 by fix-issues | the framework never said it | The npm installer wrote a settings.json missing five hook registrations the shell scripts had gained, so a project installed from the package ran without the metrics, database and build guards. | diff --git a/docs/TechieFlow-Requirements.md b/docs/TechieFlow-Requirements.md index 3691ea0..59e7bf7 100644 --- a/docs/TechieFlow-Requirements.md +++ b/docs/TechieFlow-Requirements.md @@ -54,8 +54,8 @@ The same four questions are asked for a miss in an application, against that app |---|---|---|---| | FR-01 | asks the stack questions (`TechieFlow-Stack-Questions.md` Q1 to Q8) before writing any project document, and records the answers in the Architecture document's Stack Decisions section. | fixture run: `*day1-greenfield MyDiary` without an answer set; the run stops and asks before any file under `docs/` is written. | D-4; Stack Questions §1 | | FR-02 | accepts a named answer set that fills in the stack questions, ships the .NET answer set as an option, and asks only the questions the set leaves open. | fixture run: `*day1-greenfield MyDiary` naming the DotNet answer set; only Q4 and the Q8 rendering mode are asked; the Stack Decisions table cites the set for the rest. | Stack Questions §1; Stack Defaults header | -| FR-03 | contains no language-, database-, UI-library- or host-specific instruction in any persona, task or shared rule file; such facts live only in answer sets and in a project's Stack Decisions. | script: grep persona, task and shared-rule files for `dotnet`, `Blazor`, `MAUI`, `Postgres`, `Serilog`, `xUnit`, `Dapper`, `DbUp`, `TrBlazeUI`, `Bluehost`; zero hits outside examples marked as examples. | owner 2026-09-04 | -| FR-04 | applies the two default rules of Q11 (logs under the build output folder, no unnecessary root folders) to every project unless the owner removes them. | script: after any fixture run, no log file at the repository root and no root folder outside the allowed set. | Stack Questions Q11 | +| FR-03 | contains no language-, database-, UI-library- or host-specific instruction in any persona, task or shared rule file; such facts live only in answer sets and in a project's Stack Decisions. | script (built 2026-09-07): `bash tests/requirements/run.sh` runs the grep over every persona and task. **It fails today** (`MISS-TechieFlow-20260907-18`): the analyst and `build-phase` hardcode routing to the TrBlazeUI and TechieRag library agents, and `day1-brownfield` names the `dotnet` answer set. The owner decides whether library routing is a legitimate exception to be worded into this line, or whether the tasks should route by prefix to whatever library agents a project has. | owner 2026-09-04 | +| FR-04 | applies the two default rules of Q11 (logs under the build output folder, no unnecessary root folders) to every project unless the owner removes them. | script (built 2026-09-07): `bash tests/requirements/run.sh` fails on a `*.log` at the repository root or a run-litter folder there. | Stack Questions Q11 | | FR-05 | enforces every rule recorded under Q11 in a project's Stack Decisions, and logs a violation as a miss. | script per rule, generated from the Stack Decisions table; for the .NET set: Dapper present and no other ORM, an `Db` project using DbUp, no `database` root folder, no NuGet package absent from the Architecture document, one configuration mechanism. | Stack Defaults Q11; TfLens incident | ### B. Day-1 and documents @@ -71,8 +71,8 @@ The same four questions are asked for a miss in an application, against that app | FR-12 | produces the checklist automatically once the owner approves the BRD; the owner never types `*split-brd`. | fixture run: after the stage 2 go-ahead on MyDiary the checklist exists without a separate command. | D-6 | | FR-13 | produces the DevGuide automatically when the build phase completes the checklist, for every project type, and refreshes it at handoff. | fixture run: `*build-phase MyDiary` to completion; `docs/MyDiary-DevGuide.md` exists at the end. | D-7 | | FR-14 | gives every human document template a strict structure (required sections in order, size budget per size class, row rules) and refuses to close a phase whose document breaks it. | script: `bash tests/doc-check/run.sh` — `tf-doc-check.sh` exits 0 on a clean generated document set, non-zero on the deliberately broken twin, and the status gate refuses to close. | How-It-Works §2 Template; Session 3 | -| FR-15 | requires every checklist row and every BRD item to carry one acceptance line of the form "when … then …" naming an observable result, of at most 30 words (target 20) and holding one behaviour, under a title in everyday words; BRD items sit under a heading per screen that opens with one plain sentence. | script: every `REQ-` row and `BRD-N` item in the fixture documents matches the pattern and the word cap; the broken twin's 42-word line fails. | D-20; owner 2026-09-06 (miss 13) | -| FR-16 | keeps one checklist per application as the single source of truth, in markdown only, and never creates dated `docs/qa/` or `docs/verify/` files or `-v2` document copies. | script: no `docs/qa/`, `docs/verify/`, `*-v2.*` or `*-Checklist.html` in any fixture. | conventions | +| FR-15 | requires every checklist row and every BRD item to carry one acceptance line of the form "when … then …" naming an observable result, of at most 30 words (target 20) and holding one behaviour, under a title in everyday words; BRD items sit under a heading per screen that opens with one plain sentence. | script: `bash tests/doc-check/run.sh` — the clean set passes and the broken twin's bundled 41-word line fails on the pattern and the word cap. | D-20; owner 2026-09-06 (miss 13) | +| FR-16 | keeps one checklist per application as the single source of truth, in markdown only, and never creates dated `docs/qa/` or `docs/verify/` files or `-v2` document copies. | script (built 2026-09-07): `bash tests/requirements/run.sh`. | conventions | | FR-17 | renders every human document to HTML by script, never by hand; configuration and agent documents are not rendered. | script: every human `docs/*.md` has a sibling `.html` newer than itself; no `.html` exists for the checklist, the Stack documents or this file. | TF-003; owner 2026-09-04 | | FR-54 | lays a Large project out by phase: BRD, checklist, UIDesign and DevGuide are one file per phase (phase 1 under the plain names, phase n as `-Pn-…`), `BRD-N` and `REQ-` ids run on across phases and are never reused, a `docs/-Phases.md` table names every phase's screens and id range, `appPhase` in `core-config.yaml` selects the phase every command works on, and the Architecture, Coding Standards, PROJECT-STATUS, UsageGuide, ProductGuide and the mockup folder stay single. | script: `bash tests/doc-check/run.sh` passes the two-phase fixture clean and fails its broken twin on a screen in two phases, an id in two phases, an id outside its phase's range and a phase without its BRD; `tf-split-brd.sh --all-phases` numbers phase 2 after phase 1. | owner 2026-09-05 (Schemas §2, §3.11, §7.2 H to L) | | FR-53 | produces mockups as one click-through set: every link and form action resolves to a mockup that exists when opened from the mockup folder, navigation is a link or a form action and never a script, every menu item leads to its screen, every stylesheet exists, every screen is reachable by clicking from the entry screen, every screen has a way out, every button navigates or shows a message, and every mockup carries `data-testid` anchors. Existing mockups in a brownfield repository are moved into `docs/mockups/` before anything is linked. | script: `tf-doc-check.sh --app ` FAILs on a broken link, an unreachable or dead-end screen, an inert button or an unanchored mockup; `tf-mockups-locate.sh` leaves no mockup outside `docs/mockups/`. | owner 2026-09-05 (MISS-TechieFlow-20260905-05); D-5 | @@ -82,11 +82,11 @@ The same four questions are asked for a miss in an application, against that app | ID | The framework … | Check | Source | |---|---|---|---| | FR-18 | builds UI requirements from the approved mockups and nothing else, and compares the built screen to its mockup before marking it implemented. | fixture run: `*build-phase MyDiary`; `tests/.artifacts/verify/screens.json` (written by `tf-verify-screens.sh`, the smoke evidence since Sitting 4c) has an entry for every screen the build touched, and `tests/.artifacts/verify/parity.json` names the mockup compared for every UI row. Reworded 2026-09-07 from MISS-TechieFlow-20260906-07: the old check read a smoke log no task wrote. | 39 partial-implementation misses; How-It-Works §3.4 | -| FR-19 | never writes `Verified` from a build, a fix or a status refresh; only an executed verify may. | script (hook exists): a write of `Verified` without a same-day verify ledger is refused. | convention; guard-verify hook | +| FR-19 | never writes `Verified` from a build, a fix or a status refresh; only an executed verify may. | script (built 2026-09-07): `bash tests/requirements/run.sh` drives `guard-verify.sh` both ways — refused with no ledger, allowed with a same-day one. | convention; guard-verify hook | | FR-20 | ends every command by rewriting PROJECT-STATUS in template shape and appending one run record. | script: after any fixture command, PROJECT-STATUS matches the template shape and `runs.jsonl` has one new line. | status gate; D-11 | | FR-21 | records a library gap in that library's feedback file and holds the feature; it never implements a workaround. | fixture run: build a MyDiary requirement needing a TrBlazeUI control that does not exist; the row is `BLOCKED-BY-LIBRARY`, the feedback file has the entry, no workaround code exists. | Stack Defaults Q11.3 | | FR-22 | starts a stopped database container itself when the database is unreachable, asks only when no container exists, and never creates its own database. | fixture run: stop the PostgreSQL container, run `*build-phase MyDiary`; the container is started and no new container or compose file appears. | Stack Defaults Q3 | -| FR-23 | writes all run-generated artefacts under `tests/.artifacts/` and never at the repository root. | script (hook exists): no root-level `test-results*`, `scripts-*` or similar after a fixture run. | TechieBlog incident | +| FR-23 | writes all run-generated artefacts under `tests/.artifacts/` and never at the repository root. | script (built 2026-09-07): `bash tests/requirements/run.sh` drives `guard-artifacts.sh`: a root `--output` and a root `mkdir` are refused, the `tests/.artifacts/` form is allowed. | TechieBlog incident | ### D. Verify @@ -95,32 +95,32 @@ The same four questions are asked for a miss in an application, against that app | FR-24 | applies the seven checks to every requirement in a fixed order and records the first that fails. | script: `bash tests/verify/run.sh` — every `gates.jsonl` record the fixture verify writes carries a gate value from the fixed list or none. | How-It-Works §6.2 | | FR-25 | verifies against the acceptance line and the mockup, and states in the remark what was observed, so a `Verified` row can be re-derived by a reader. | review, to become a script: sample ten verified rows across fixtures; each remark names the observation. | 63 misses classified insufficient-verify-method | | FR-26 | has a verify task of at most 4,000 words, with every mechanical step in a script. | script: `bash tests/mirror/run.sh` counts `verify-phase.md` and fails above 4,000 words (built 2026-09-07; it is 964). | D-8 | -| FR-27 | never reports a file or tool as "not present" without trying its literal path, because the framework folder is invisible to search. | script: the phrase "not present" in a checklist remark is refused unless the remark also names the path tried. | D-21 | +| FR-27 | never reports a file or tool as "not present" without trying its literal path, because the framework folder is invisible to search. | script: `bash tests/doc-check/run.sh` — the broken twin plants a Remarks cell that says "not present" without a path, and the checker refuses it. | D-21 | ### E. Bugs and misses | ID | The framework … | Check | Source | |---|---|---|---| -| FR-28 | never edits source or spawns builders during `*triage-issues`. | script: no file under `src/` or `tests/` changes during a fixture triage run. | convention; How-It-Works §3.7 | +| FR-28 | never edits source or spawns builders during `*triage-issues`. | script: `bash tests/bugs/run.sh` — a triage run over the fixture leaves `src/` untouched, and a planted code edit during triage is reported and logged as `instruction-ignored`. | convention; How-It-Works §3.7 | | FR-29 | records a miss automatically from triage (discovery cost) and from fix (fix cost); the owner never types `*log-miss` for a bug that went through either. | fixture run: triage then fix one bug on MyDiary; `misses.jsonl` gains a `miss` and a `miss-fix` with no manual log command. | D-15 | | FR-30 | offers one command that runs the owner's bug sequence end to end in YOLO mode: compare screens to mockups, triage, log discovery cost, fix, log fix cost, metrics, with a summary per step. | fixture run: `*triage-and-fix MyDiary `; the final summary has six sections. | D-16 | | FR-31 | stores the owner's one-sentence description of every miss in a human-readable file beside the record. | script (built 2026-09-07): `tf-emit.sh` rebuilds `docs/-Misses.md` and its HTML from the stream after every write to it; `bash tests/bugs/run.sh` checks that the row count equals the record count and that the sentence, the row and whose gap are in the row. | D-10 | | FR-32 | sorts every miss with the four questions of §3 and records the answer. | script (built 2026-09-07): `tf-log-miss.sh` refuses a miss without `--sort` and prints the four questions; `tf-triage.sh` defaults it; `tf-emit.sh --amend sort ` sorts an older record once and never twice; `tf-metrics.sh` reports the distribution over the records that carry it. `bash tests/bugs/run.sh`. | §3 | -| FR-33 | records issues found by people in UAT as misses, never as reviews. | script: every record from `*triage-issues` is of kind `miss`. | owner 2026-09-04 | +| FR-33 | records issues found by people in UAT as misses, never as reviews. | script: `bash tests/bugs/run.sh` — every record the triage run writes to the miss stream is of kind `miss`, one per row. | owner 2026-09-04 | ### F. Telemetry | ID | The framework … | Check | Source | |---|---|---|---| -| FR-34 | emits one run record for every command, including day-1, mockups, DevGuide, ProductGuide and the idea-stage commands. | script: after each fixture command, `runs.jsonl` has a record with that command's name. | D-11; D-13 | -| FR-35 | records on every run whether YOLO mode was on. | script: every new run record carries `yolo: true|false`. | D-12 | +| FR-34 | emits one run record for every command, including day-1, mockups, DevGuide, ProductGuide and the idea-stage commands. | script (built 2026-09-07): `bash tests/requirements/run.sh` fails when a command task wires no run record at all. **It fails today**: `create-doc`, `generate-html`, `facilitate-brainstorming-session` and `create-deep-research-prompt` write none, which is the idea-stage gap D-13 named and FR-60 still carries. A real run writing the record stays a fixture check. | D-11; D-13 | +| FR-35 | records on every run whether YOLO mode was on. | script (built 2026-09-07): `bash tests/requirements/run.sh` emits a record without the field and checks the appended one carries it. | D-12 | | FR-36 | records the outcome of every owner review as a record of kind `review` on the misses stream, named by its phase (`day1-review`, `build-review`, `verify-review`, `handoff-review`), carrying the number of corrections given, and the cost of producing the reviewed output and of applying the corrections, both copied by the emitter from the two runs the record names. | script: the emitter refuses a review without a phase from the list or without a corrections count, and copies the costs from the runs named (built 2026-09-06); fixture run: MyDiary stage 2 after the owner's review of stage 1 leaves a `day1-review` record. | D-17; owner 2026-09-04; built Sitting 4b | -| FR-37 | records framework maintenance work under the command value `framework-reset`. | script: the schema lists the value; the report accepts it. | D-19 | +| FR-37 | records framework maintenance work under the command value `framework-reset`. | script (built 2026-09-07): `bash tests/requirements/run.sh` checks that `SCHEMA.md` lists the value and that the report carries it. It failed on its first run: the schema had never gained `framework-reset`, nor four other values its own tasks were writing (`MISS-TechieFlow-20260907-17`). | D-19 | | FR-38 | never merges provenance in a report: live with backfilled, or one project type with another. | script (exists in `tf-metrics.sh`): the report prints separate figures. | schema §0 | | FR-39 | never blocks, fails or changes a verdict because a telemetry write failed. | review: `tf-emit.sh` exits 0 on every path. | schema | | FR-55 | records a run's `ended` as the moment the record is written, never a value the agent guessed: an `ended` in the future or before `started` is replaced with now and the duration recomputed; `started` comes from the command marker `tf-phase.sh start` writes at step 0 of every task when the record leaves it out. | script: emit a run record with `ended` an hour ahead on a fixture; the appended record carries now; a record without `started` carries the marker's time. | MISS-TechieFlow-20260905-11 | | FR-56 | writes to an application's database only from `*build-phase` and `*fix-issues`, and only through the migration path the Stack decisions name; a direct SQL write through a client is refused from every command, and a migration runner is refused unless the command marker says build-phase or fix-issues. | script (hook `guard-db.sh`, both harnesses): an SQL update is refused with and without a marker; a migration runner is refused under a day-1 marker and allowed under a build marker; a select and a build pass. | MISS-TechieFlow-20260905-09; owner 2026-09-05 | -| FR-57 | checks the Architecture's Stack decisions table for a row per question 1 to 8 and 11, and refuses an app whose Solution structure names `.App` or has no project named exactly the app. | script: the broken fixture fails on a missing Q3 row and on `MyDiary.App`. | MISS-TechieFlow-20260905-16; owner 2026-09-06 | +| FR-57 | checks the Architecture's Stack decisions table for a row per question 1 to 8 and 11, and refuses an app whose Solution structure names `.App` or has no project named exactly the app. | script: `bash tests/doc-check/run.sh` — the broken twin carries a missing stack row and an `.App` head, and both fail. | MISS-TechieFlow-20260905-16; owner 2026-09-06 | ### G. Harnesses @@ -150,10 +150,10 @@ The same four questions are asked for a miss in an application, against that app | ID | The framework … | Check | Source | |---|---|---|---| -| FR-48 | is published as an npm package, `@techierathore/techieflow`, installable into any project with one command, `npx @techierathore/techieflow@latest install`, and updatable with `… update`, without adding the framework as an application dependency. | script: on a clean fixture clone, the install command produces the same file set as `scaffold-brownfield.sh` (diff of the two results is empty) and leaves no `node_modules`, `package.json` or lock file behind. | D-22 | -| FR-49 | installs for both harnesses from the one package: the Claude Code mirror and settings, and the OpenCode registrations. | script: after install, `.claude/commands/TechieFlow/` is byte-identical to `.tfcore/`, and every `opencode.jsonc` file reference resolves. | D-22; FR-40 | -| FR-50 | is versioned through GitHub releases and published by a pipeline that runs automated checks first: mirror parity, OpenCode reference resolution, `bash -n` on every script, the installer's own tests, and a dry-run pack. | script: the release workflow fails when any check fails; the published package version equals the release tag. | D-22; Playbook release process | -| FR-51 | keeps the shell scripts (`scaffold-*.sh`, `update-framework.sh`) working from a local clone, and the installer produces the same result, so both routes stay valid. | script: the FR-48 diff, run from both routes. | D-22 | +| FR-48 | is published as an npm package, `@techierathore/techieflow`, installable into any project with one command, `npx @techierathore/techieflow@latest install`, and updatable with `… update`, without adding the framework as an application dependency. | script: `npm run test:install` — it installs by each route into identical folders and compares every path, its content and its executable bit, then checks no npm footprint is left. Run it on a normal filesystem. | D-22 | +| FR-49 | installs for both harnesses from the one package: the Claude Code mirror and settings, and the OpenCode registrations. | script: `npm run test:install` checks the mirror and both `opencode.jsonc` files after an install by each route. | D-22; FR-40 | +| FR-50 | is versioned through GitHub releases and published by a pipeline that runs automated checks first: mirror parity, OpenCode reference resolution, `bash -n` on every script, the installer's own tests, and a dry-run pack. | script (built 2026-09-07): `bash tests/requirements/run.sh` reads `.github/workflows/release.yml` and fails unless `npm run validate` and `npm run test:install` both run before `npm publish`. The tag-equals-version half remains a review. | D-22; Playbook release process | +| FR-51 | keeps the shell scripts (`scaffold-*.sh`, `update-framework.sh`) working from a local clone, and the installer produces the same result, so both routes stay valid. | script: `npm run test:install` — the same diff, from both routes. | D-22 | | FR-63 | carries every change that alters what a project receives into **both** routes in the same pass: the shell scripts and the npm installer. That includes a new or removed hook registration in `.claude/settings.json`, a new folder under `.tfcore/`, and any change to how an existing project's files are refreshed. | script: `npm run test:install` installs by each route into identical folders and compares every path, its content and its executable bit; it failed on five checks when a hook registration and the `standards` folder were in the shell scripts only. Run it on a normal filesystem: a Windows mount reports every file executable and produces one false difference. | MISS-TechieFlow-20260907-13 and -14 (both sorted `unsaid`); D-22 | | FR-52 | ships an Installation document that a person outside the owner's machines can follow to a working project in under ten minutes. | review, then fixture run: a fresh machine with Node installed, following the document only, reaches a working `*day1-greenfield` on MyDiary. | D-22 | diff --git a/docs/TfLens-Metrics-Update-Prompt.html b/docs/TfLens-Metrics-Update-Prompt.html new file mode 100644 index 0000000..2039ee1 --- /dev/null +++ b/docs/TfLens-Metrics-Update-Prompt.html @@ -0,0 +1,412 @@ + + + + + +TfLens — what to add so the framework's problems show up + + + + + +
    + +
    +

    TfLens — what to add so the framework's problems show up

    +
    Rendered 2026-09-07 · source TfLens-Metrics-Update-Prompt.md
    + +
    +
    Contents
    +
      +
    1. 1. Why this exists
      2. +
    2. 2. The one new field that changes what the dashboard is for
      3. +
    3. sort on a miss record — whose gap was it
      4. +
    4. 3. Four more things now in the streams
      5. +
    5. 3.1 what — the sentence a human can read
      6. +
    6. 3.2 kind: "review" — what an owner review cost
      7. +
    7. 3.3 kind: "miss-amend" — a field completed later
      8. +
    8. 3.4 req_class: "FR" on a gate record
      9. +
    9. 4. Three reader-side rules that change numbers TfLens already shows
      10. +
    10. 4.1 Key a requirement by project and id
      11. +
    11. 4.2 Derive a run's duration when the record omits it
      12. +
    12. 4.3 Derive attempt when the record omits it
      13. +
    13. 5. Two vocabulary changes
      14. +
    14. 6. What must not be done
      15. +
    15. 7. How the team proves each item
      16. +
    16. 8. If anything here is wrong
      17. +
    +
    +
    MissFoundClosedWhose gapWhat went wrong
    MISS-TechieFlow-20260907-172026-09-07 by owner2026-09-07 by fix-issuesthe check was too weakThe telemetry schema never gained the command values its own tasks were writing, so 27 run records across the estate carried a cmd the schema does not list and nothing noticed for four days.
    MISS-TechieFlow-20260907-162026-09-07 by owner2026-09-07 by fix-issuessaid and ignoredThe framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all.
    MISS-TechieFlow-20260907-152026-09-07 by owner2026-09-07 by fix-issuesthe check was too weakA run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved.
    MISS-TechieFlow-20260907-142026-09-07 by owner2026-09-07 by fix-issuesthe framework never said itThe npm installer's framework subfolder list left out standards, so a project migrated from the old layout came out with no coding standards file.
    MISS-TechieFlow-20260907-132026-09-07 by owner2026-09-07 by fix-issuesthe framework never said itThe npm installer wrote a settings.json missing five hook registrations the shell scripts had gained, so a project installed from the package ran without the metrics, database and build guards.
    + + + + + + + +
    PurposeHand this to whoever works on TfLens. It says exactly what changed in the telemetry between 2026-08-31 and 2026-09-07, what TfLens must add to show it, and the three reader-side rules that change numbers TfLens already publishes.
    AudienceThe TfLens team. No knowledge of TechieFlow's internals is assumed.
    Source of truth.tfcore/telemetry/SCHEMA.md in any repository carrying the framework. Where this document and the schema disagree, the schema wins and this document is wrong.
    Written2026-09-07, at the close of the TechieFlow reset.
    +
    +

    1. Why this exists#

    +

    TfLens reads the framework's telemetry and turns it into pages. Between 2026-08-31 and 2026-09-07 the framework was rewritten, and the streams gained fields that answer a question the dashboard cannot answer today: not "what broke" but "whose fault was it, and would a check have caught it". Everything below is already being written into docs/metrics/*.jsonl in every project. None of it needs a framework change; it needs a reader.

    +

    One number is also wrong today and will stay wrong until a reader-side rule changes. That is §4, and it matters more than any new page.

    +
    +

    2. The one new field that changes what the dashboard is for#

    +

    sort on a miss record — whose gap was it#

    +

    Every miss now carries the answer to four questions asked in order, stored in one field:

    + + + + + + + + +
    sortMeansThe fixed response
    specthe project's own specification did not say itfix the checklist line; the framework is untouched
    unsaidthe framework never said it anywhereadd one requirement line, plus a check
    weak-checkthere was a check and it did not catch itfix the check, not the prose
    ignoredit was written down and ignored anywaymake it a hook or a script, or delete the rule
    +

    Why it is the important one. Across the framework's own 126 misses, roughly half sorted weak-check, a quarter unsaid and a quarter ignored. That distribution says the dominant failure is not "nobody wrote the rule" but "the check was too weak to catch it" — which is a completely different remedy from the one a class-only chart implies. A dashboard that shows miss_class but not sort shows what broke and hides what to do about it.

    +

    What to build. A distribution of sort beside the existing class distribution, per project and combined, and a filter. On a project page, the sentence for each bucket in plain words rather than the raw value, because the raw values mean nothing to a reader who has not read the schema.

    +

    Honesty rule. The field arrived on 2026-09-07 and older records do not carry it. Report the count that carries it as the denominator — "33 of 65 sorted" — never as a percentage of all misses. A miss written before the field existed is not "unsorted by choice", and the two must not be pooled.

    +
    +

    3. Four more things now in the streams#

    +

    3.1 what — the sentence a human can read#

    +

    Every miss carries a one-sentence description in the owner's words. The framework also rebuilds docs/<App>-Misses.md and its HTML from the stream after every write, so a readable list already exists in each project. TfLens should show the sentence on the miss row: a miss identified only by class and phase cannot be recognised a month later.

    +

    Never store or re-publish requirement text from a checklist. The what sentence is written for this purpose and is the only prose that travels.

    +

    3.2 kind: "review" — what an owner review cost#

    +

    A new record kind on the misses stream (schema §5.5.9). One per phase that ends in an owner review, carrying the phase (day1-review, build-review, verify-review, handoff-review), how many corrections came back, and the cost of producing the reviewed output and of applying the corrections — both copied by the emitter from the two runs the record names, never typed.

    +

    This is the only record that prices a specification defect. A dashboard that shows build and verify costs but not review costs tells the reader that documents are free.

    +

    3.3 kind: "miss-amend" — a field completed later#

    +

    Also on the misses stream. It fills a field that was left null and never overwrites one that is not. A reader that ignores amendments sees nothing false, but it sees less: the framework's own stream has 53 of them, mostly completing sort on older records. Fold them in when computing a distribution, and say how many were folded.

    +

    3.4 req_class: "FR" on a gate record#

    +

    The framework now grades itself against its own 63 requirement lines and writes one gate record per line, with req_class: "FR". These are not application requirements and must never pool with UI, FN, RAG or NFR: a framework rule and a screen are different units. Segment them, or exclude them, but do not average them together.

    +
    +

    4. Three reader-side rules that change numbers TfLens already shows#

    +

    These are not new fields. They are arithmetic the reader must do, and TfLens is currently doing two of them differently. Each was a real defect in the framework's own report before it was fixed there.

    +

    4.1 Key a requirement by project and id#

    +

    Every project has a REQ-UI-001. A rollup keyed on req_id alone counts TfLens's and TechieBlog's as one requirement. The framework's own combined first-pass rate read 72% under that bug and 48% once keyed on (project, req_id). If TfLens publishes any cross-project figure keyed on the id alone, that figure is wrong by roughly the same margin.

    +

    4.2 Derive a run's duration when the record omits it#

    +

    duration_s was added after some records were written. Those records carry started and ended but no duration, and a reader that sums duration_s counts them as zero time while still counting their tokens — so a phase's time is a sum over some of its runs and its tokens a sum over others. The framework's reset reported 16h49m under that bug against a true 55h57m. Derive it from started and ended; where ended is absent too, the schema defines it as the moment the record was written, which is ts. Print how many were derived.

    +

    4.3 Derive attempt when the record omits it#

    +

    First-pass rate counts records with attempt == 1. attempt is defined as 1 plus the number of prior live records for the same requirement in the same project — a count over the stream, not a judgement. A record without it drops out of the rate entirely: the framework's own 12 requirement verdicts, all passing, reported a first-pass rate of 0%. Derive it in stream order for records that lack it, and never touch one that carries it.

    +
    +

    5. Two vocabulary changes#

    +
      +
    • harness: "codex" is retired. The Codex adapter was removed on 2026-09-07. Nothing writes that value any more; records that carry it stay valid and must still render. Do not drop them, and do not add Codex to any new filter. +
      • +
    • The cmd vocabulary grew: metrics-report, generate-html, render-workflow-docs, triage-and-fix, and framework-reset for framework maintenance. A dashboard that whitelists command names will silently hide 27 records that already exist across the estate. Prefer showing an unknown value to dropping it. +
      • +
    +
    +

    6. What must not be done#

    +

    These are the framework's own standing rules for any report built on these streams, and they apply to TfLens.

    +
      +
    • Provenance never merges. Live records never pool with backfilled ones; app, library, docs and framework project types never pool with each other; a miss whose attribution is inferred never enters a per-model or per-phase rate. Data on the wrong side of a boundary may sit in an adjacent labelled column, never summed with the figure beside it. +
      • +
    • Every exclusion is printed with the figure it bounds. An exclusion the reader cannot see is indistinguishable from a bug. +
      • +
    • Fewer than three supporting records is insufficient data, not a number. +
      • +
    • No dollar figure is ever estimated. Real dollars exist only on OpenCode records. Claude Code carries cost_usd: null permanently; report tokens and say why. No rate card, ever. +
      • +
    • A per-model rate is observational, not causal. Which model gets the hard work is not random, and the page must say so once. +
      • +
    +
    +

    7. How the team proves each item#

    +

    Nothing here is done until it is shown against real data. Every project under /mnt/c/1MyCode and /mnt/c/3AIGenCode carries live streams; TfLens itself has 161 miss records and 809 gate records.

    + + + + + + + + + + + + +
    ItemThe proof
    sort distributionThe three buckets appear with their denominator, and the count of records predating the field is stated separately.
    what sentenceA miss row shows the sentence, and no requirement text appears anywhere on the page.
    review recordsA phase with a review shows corrections given, cost to produce and cost to correct, all copied not computed.
    miss-amendA distribution states how many fields were completed by an amendment.
    req_class: FRFramework requirement verdicts appear in their own segment and in no combined application figure.
    Rollup keyingThe combined first-pass rate is recomputed; if it moves by roughly 20 points, the old figure was keyed on the id alone.
    Duration derivationA phase's total time and its token total cover the same set of runs, and the page says how many durations were derived.
    attempt derivationA set of records that all passed reports a first-pass rate of 100%, not 0%.
    +
    +

    8. If anything here is wrong#

    +

    Say so, with the record that disproves it. The framework's own report was wrong about three of these until someone checked, and each correction came from reading the records rather than the documentation. .tfcore/telemetry/SCHEMA.md is the contract; this page is a summary of it written on 2026-09-07 and will age.

    + + + + + + + + diff --git a/docs/TfLens-Metrics-Update-Prompt.md b/docs/TfLens-Metrics-Update-Prompt.md new file mode 100644 index 0000000..41273cd --- /dev/null +++ b/docs/TfLens-Metrics-Update-Prompt.md @@ -0,0 +1,121 @@ +# TfLens — what to add so the framework's problems show up + +| | | +|---|---| +| Purpose | Hand this to whoever works on TfLens. It says exactly what changed in the telemetry between 2026-08-31 and 2026-09-07, what TfLens must add to show it, and the three reader-side rules that change numbers TfLens already publishes. | +| Audience | The TfLens team. No knowledge of TechieFlow's internals is assumed. | +| Source of truth | `.tfcore/telemetry/SCHEMA.md` in any repository carrying the framework. Where this document and the schema disagree, the schema wins and this document is wrong. | +| Written | 2026-09-07, at the close of the TechieFlow reset. | + +--- + +## 1. Why this exists + +TfLens reads the framework's telemetry and turns it into pages. Between 2026-08-31 and 2026-09-07 the framework was rewritten, and the streams gained fields that answer a question the dashboard cannot answer today: **not "what broke" but "whose fault was it, and would a check have caught it".** Everything below is already being written into `docs/metrics/*.jsonl` in every project. None of it needs a framework change; it needs a reader. + +One number is also wrong today and will stay wrong until a reader-side rule changes. That is §4, and it matters more than any new page. + +--- + +## 2. The one new field that changes what the dashboard is for + +### `sort` on a miss record — whose gap was it + +Every miss now carries the answer to four questions asked in order, stored in one field: + +| `sort` | Means | The fixed response | +|---|---|---| +| `spec` | the project's own specification did not say it | fix the checklist line; the framework is untouched | +| `unsaid` | the framework never said it anywhere | add one requirement line, plus a check | +| `weak-check` | there was a check and it did not catch it | fix the check, not the prose | +| `ignored` | it was written down and ignored anyway | make it a hook or a script, or delete the rule | + +**Why it is the important one.** Across the framework's own 126 misses, roughly half sorted `weak-check`, a quarter `unsaid` and a quarter `ignored`. That distribution says the dominant failure is not "nobody wrote the rule" but "the check was too weak to catch it" — which is a completely different remedy from the one a class-only chart implies. A dashboard that shows `miss_class` but not `sort` shows what broke and hides what to do about it. + +**What to build.** A distribution of `sort` beside the existing class distribution, per project and combined, and a filter. On a project page, the sentence for each bucket in plain words rather than the raw value, because the raw values mean nothing to a reader who has not read the schema. + +**Honesty rule.** The field arrived on 2026-09-07 and older records do not carry it. Report the count that carries it as the denominator — "33 of 65 sorted" — never as a percentage of all misses. A miss written before the field existed is not "unsorted by choice", and the two must not be pooled. + +--- + +## 3. Four more things now in the streams + +### 3.1 `what` — the sentence a human can read + +Every miss carries a one-sentence description in the owner's words. The framework also rebuilds `docs/-Misses.md` and its HTML from the stream after every write, so a readable list already exists in each project. TfLens should show the sentence on the miss row: a miss identified only by class and phase cannot be recognised a month later. + +**Never store or re-publish requirement text** from a checklist. The `what` sentence is written for this purpose and is the only prose that travels. + +### 3.2 `kind: "review"` — what an owner review cost + +A new record kind on the misses stream (schema §5.5.9). One per phase that ends in an owner review, carrying the phase (`day1-review`, `build-review`, `verify-review`, `handoff-review`), how many corrections came back, and the cost of producing the reviewed output and of applying the corrections — both **copied by the emitter** from the two runs the record names, never typed. + +This is the only record that prices a specification defect. A dashboard that shows build and verify costs but not review costs tells the reader that documents are free. + +### 3.3 `kind: "miss-amend"` — a field completed later + +Also on the misses stream. It fills a field that was left `null` and **never overwrites** one that is not. A reader that ignores amendments sees nothing false, but it sees less: the framework's own stream has 53 of them, mostly completing `sort` on older records. Fold them in when computing a distribution, and say how many were folded. + +### 3.4 `req_class: "FR"` on a gate record + +The framework now grades itself against its own 63 requirement lines and writes one gate record per line, with `req_class: "FR"`. These are **not** application requirements and must never pool with `UI`, `FN`, `RAG` or `NFR`: a framework rule and a screen are different units. Segment them, or exclude them, but do not average them together. + +--- + +## 4. Three reader-side rules that change numbers TfLens already shows + +These are not new fields. They are arithmetic the reader must do, and TfLens is currently doing two of them differently. Each was a real defect in the framework's own report before it was fixed there. + +### 4.1 Key a requirement by project **and** id + +Every project has a `REQ-UI-001`. A rollup keyed on `req_id` alone counts TfLens's and TechieBlog's as one requirement. The framework's own combined first-pass rate read **72%** under that bug and **48%** once keyed on `(project, req_id)`. If TfLens publishes any cross-project figure keyed on the id alone, that figure is wrong by roughly the same margin. + +### 4.2 Derive a run's duration when the record omits it + +`duration_s` was added after some records were written. Those records carry `started` and `ended` but no duration, and a reader that sums `duration_s` counts them as **zero time while still counting their tokens** — so a phase's time is a sum over some of its runs and its tokens a sum over others. The framework's reset reported **16h49m** under that bug against a true **55h57m**. Derive it from `started` and `ended`; where `ended` is absent too, the schema defines it as the moment the record was written, which is `ts`. Print how many were derived. + +### 4.3 Derive `attempt` when the record omits it + +First-pass rate counts records with `attempt == 1`. `attempt` is *defined* as 1 plus the number of prior live records for the same requirement in the same project — a count over the stream, not a judgement. A record without it drops out of the rate entirely: the framework's own 12 requirement verdicts, all passing, reported a first-pass rate of **0%**. Derive it in stream order for records that lack it, and never touch one that carries it. + +--- + +## 5. Two vocabulary changes + +- **`harness: "codex"` is retired.** The Codex adapter was removed on 2026-09-07. Nothing writes that value any more; records that carry it stay valid and must still render. Do not drop them, and do not add Codex to any new filter. +- **The `cmd` vocabulary grew**: `metrics-report`, `generate-html`, `render-workflow-docs`, `triage-and-fix`, and `framework-reset` for framework maintenance. A dashboard that whitelists command names will silently hide 27 records that already exist across the estate. Prefer showing an unknown value to dropping it. + +--- + +## 6. What must not be done + +These are the framework's own standing rules for any report built on these streams, and they apply to TfLens. + +- **Provenance never merges.** Live records never pool with backfilled ones; `app`, `library`, `docs` and `framework` project types never pool with each other; a miss whose attribution is inferred never enters a per-model or per-phase rate. Data on the wrong side of a boundary may sit in an adjacent labelled column, never summed with the figure beside it. +- **Every exclusion is printed with the figure it bounds.** An exclusion the reader cannot see is indistinguishable from a bug. +- **Fewer than three supporting records is `insufficient data`**, not a number. +- **No dollar figure is ever estimated.** Real dollars exist only on OpenCode records. Claude Code carries `cost_usd: null` permanently; report tokens and say why. No rate card, ever. +- **A per-model rate is observational, not causal.** Which model gets the hard work is not random, and the page must say so once. + +--- + +## 7. How the team proves each item + +Nothing here is done until it is shown against real data. Every project under `/mnt/c/1MyCode` and `/mnt/c/3AIGenCode` carries live streams; TfLens itself has 161 miss records and 809 gate records. + +| Item | The proof | +|---|---| +| `sort` distribution | The three buckets appear with their denominator, and the count of records predating the field is stated separately. | +| `what` sentence | A miss row shows the sentence, and no requirement text appears anywhere on the page. | +| `review` records | A phase with a review shows corrections given, cost to produce and cost to correct, all copied not computed. | +| `miss-amend` | A distribution states how many fields were completed by an amendment. | +| `req_class: FR` | Framework requirement verdicts appear in their own segment and in no combined application figure. | +| Rollup keying | The combined first-pass rate is recomputed; if it moves by roughly 20 points, the old figure was keyed on the id alone. | +| Duration derivation | A phase's total time and its token total cover the same set of runs, and the page says how many durations were derived. | +| `attempt` derivation | A set of records that all passed reports a first-pass rate of 100%, not 0%. | + +--- + +## 8. If anything here is wrong + +Say so, with the record that disproves it. The framework's own report was wrong about three of these until someone checked, and each correction came from reading the records rather than the documentation. `.tfcore/telemetry/SCHEMA.md` is the contract; this page is a summary of it written on 2026-09-07 and will age. diff --git a/docs/metrics/METRICS.html b/docs/metrics/METRICS.html index 4e082d2..4896a6c 100644 --- a/docs/metrics/METRICS.html +++ b/docs/metrics/METRICS.html @@ -150,7 +150,7 @@

    TechieFlow — Development Metrics

    StreamRecordsSpan runs.jsonl422026-08-28 → 2026-09-07 -gates.jsonl122026-09-07 (the framework's own requirement lines, graded for the first time) +gates.jsonl412026-09-07 (two grading passes over the framework's own requirement lines) sessions.jsonl292026-08-20 → 2026-09-07 commits.jsonl522026-06-25 → 2026-09-07 misses.jsonl121 miss + 88 miss-fix + 53 miss-amend2026-08-28 → 2026-09-07 @@ -158,20 +158,27 @@

    TechieFlow — Development Metrics

    1. First-pass rate — the framework's own requirements#

    -

    100% of the 12 lines that can be graded, 12 of 63. Every one passed on its first recorded verdict.

    -

    That number needs its denominator and its date, or it flatters the framework:

    +

    93%: 27 of the 29 lines that can be graded passed on their first recorded verdict. Two failed, and both failures are real.

    - - - - + + + +
    Requirement lines in docs/TechieFlow-Requirements.md63
    Graded by a check that runs today12
    Passed12
    Failed0
    Ungraded51
    Graded by a check that runs today29
    Passed27
    Failed2
    Ungraded34
    -

    Why only 12. A line is graded only when its own Check column names something runnable — a self-test, a script, an npm script — and that artefact is run for the verdict. Of the other 51: 14 name a fixture run (a real command on a real project, which no automated pass can stand in for), 5 name a review by a person, 1 needs a normal filesystem (FR-63, which a Windows mount cannot grade honestly), 3 are script candidates that were never written, and 28 describe a script in prose that has no runnable artefact behind it. Those 28 are the finding: the framework wrote down how it would check itself and then did not build the check.

    -

    What this figure is not. These 12 lines were graded for the first time on 2026-09-07, at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: 121 misses logged during the same period (§5). Read the two together or neither.

    +

    The two failures, neither of them new, both invisible until a check existed to say so:

    +
      +
    • FR-03, technology neutrality. The line says no persona or task names a language, database, UI library or host. The analyst and build-phase hardcode routing to the TrBlazeUI and TechieRag library agents, and day1-brownfield names the dotnet answer set. Either library routing is a legitimate exception and the line must say so, or the tasks must route by requirement prefix to whatever library agents a project has. That is the owner's call (MISS-TechieFlow-20260907-18). +
      • +
    • FR-34, a run record for every command. Four tasks wire none: create-doc, generate-html, facilitate-brainstorming-session and create-deep-research-prompt. This is the idea-stage gap D-13 named in Session 1 and FR-60 still carries; the check now states it in a number rather than in prose. +
      • +
    +

    Why 29 and not 63. A line is graded only when something runnable proves it, and that artefact is run for the verdict. The other 34: 14 need a fixture run (a real command on a real project, which no automated pass stands in for), 5 need a review by a person, 3 are script candidates nobody built, 1 needs a filesystem this machine cannot provide (FR-63, which a Windows mount cannot grade honestly), and 11 still describe a script in prose with nothing behind it.

    +

    On 2026-09-07 that last group was 28. Checks were built for 10 of them, five more were repointed at the self-test whose planted defect already proved them, and three at the installer test. Coverage went from 12 lines to 29 in one pass, and it immediately found the two failures above plus a third defect: the telemetry schema had never listed framework-reset or four other command values its own tasks were writing (MISS-TechieFlow-20260907-17).

    +

    What this figure is not. These lines were graded for the first time at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: 126 misses logged (§5). Read the two together or neither.

    Until 2026-09-07 this section read "no data", on the reasoning that the framework had no checklist to verify. That was wrong: it has had 63 requirement lines since Session 2, named in every session restart prompt. Logged as MISS-TechieFlow-20260907-16.

    2. Gate catch distribution#

    No data, and here the original reasoning does hold: a gate distribution answers "which of the seven checks caught the failure", and those checks — build, acceptance, data, visual, assets, speed, standards — are applied to a running application's screens. This repository has none. All 12 framework verdicts passed, so there is no failure to attribute in any case.

    @@ -279,7 +286,7 @@

    6c. Subagent fan-out

    Not observed on any of the 14 reset runs. Every one carried a main-scope window, which never reads the subagent transcripts, so a zero here means not looked at, not none ran. Three runs declared an explore subagent in their own emit; the declared figure is kept beside the measured one and never merged with it.

    7. What is missing#

      -
    • 51 of the framework's 63 requirement lines are ungraded — 28 describe a script nobody built, 14 need a fixture run, 5 need a review, 3 are unbuilt candidates and 1 needs a filesystem this machine cannot provide. That is the largest gap on this page, and it is a gap in the framework's own verification, not in its telemetry. +
    • 34 of the framework's 63 requirement lines are ungraded — 14 need a fixture run, 11 still describe a script nobody built, 5 need a review, 3 are unbuilt candidates and 1 needs a filesystem this machine cannot provide. That is the largest gap on this page, and it is a gap in the framework's own verification, not in its telemetry.
    • No gate catch distribution or escape rate, because both describe a running application's screens and this repository has none.
      • diff --git a/docs/metrics/METRICS.md b/docs/metrics/METRICS.md index 641653c..d450c45 100644 --- a/docs/metrics/METRICS.md +++ b/docs/metrics/METRICS.md @@ -12,7 +12,7 @@ This repository is the framework itself, not an application. It **does** have a | Stream | Records | Span | |---|---|---| | `runs.jsonl` | 42 | 2026-08-28 → 2026-09-07 | -| `gates.jsonl` | 12 | 2026-09-07 (the framework's own requirement lines, graded for the first time) | +| `gates.jsonl` | 41 | 2026-09-07 (two grading passes over the framework's own requirement lines) | | `sessions.jsonl` | 29 | 2026-08-20 → 2026-09-07 | | `commits.jsonl` | 52 | 2026-06-25 → 2026-09-07 | | `misses.jsonl` | 121 miss + 88 miss-fix + 53 miss-amend | 2026-08-28 → 2026-09-07 | @@ -21,21 +21,26 @@ This repository is the framework itself, not an application. It **does** have a ## 1. First-pass rate — the framework's own requirements -**100% of the 12 lines that can be graded, 12 of 63.** Every one passed on its first recorded verdict. - -That number needs its denominator and its date, or it flatters the framework: +**93%: 27 of the 29 lines that can be graded passed on their first recorded verdict.** Two failed, and both failures are real. | | | |---|---| | Requirement lines in `docs/TechieFlow-Requirements.md` | 63 | -| Graded by a check that runs today | 12 | -| Passed | 12 | -| Failed | 0 | -| Ungraded | 51 | +| Graded by a check that runs today | 29 | +| Passed | 27 | +| Failed | 2 | +| Ungraded | 34 | + +**The two failures**, neither of them new, both invisible until a check existed to say so: + +- **FR-03, technology neutrality.** The line says no persona or task names a language, database, UI library or host. The analyst and `build-phase` hardcode routing to the TrBlazeUI and TechieRag library agents, and `day1-brownfield` names the `dotnet` answer set. Either library routing is a legitimate exception and the line must say so, or the tasks must route by requirement prefix to whatever library agents a project has. That is the owner's call (`MISS-TechieFlow-20260907-18`). +- **FR-34, a run record for every command.** Four tasks wire none: `create-doc`, `generate-html`, `facilitate-brainstorming-session` and `create-deep-research-prompt`. This is the idea-stage gap D-13 named in Session 1 and FR-60 still carries; the check now states it in a number rather than in prose. + +**Why 29 and not 63.** A line is graded only when something runnable proves it, and that artefact is run for the verdict. The other 34: **14 need a fixture run** (a real command on a real project, which no automated pass stands in for), **5 need a review** by a person, **3 are script candidates nobody built**, **1 needs a filesystem this machine cannot provide** (`FR-63`, which a Windows mount cannot grade honestly), and **11 still describe a script in prose with nothing behind it**. -**Why only 12.** A line is graded only when its own Check column names something runnable — a self-test, a script, an npm script — and that artefact is run for the verdict. Of the other 51: **14 name a fixture run** (a real command on a real project, which no automated pass can stand in for), **5 name a review** by a person, **1 needs a normal filesystem** (`FR-63`, which a Windows mount cannot grade honestly), **3 are script candidates that were never written**, and **28 describe a script in prose that has no runnable artefact behind it**. Those 28 are the finding: the framework wrote down how it would check itself and then did not build the check. +On 2026-09-07 that last group was 28. Checks were built for 10 of them, five more were repointed at the self-test whose planted defect already proved them, and three at the installer test. **Coverage went from 12 lines to 29 in one pass**, and it immediately found the two failures above plus a third defect: the telemetry schema had never listed `framework-reset` or four other command values its own tasks were writing (`MISS-TechieFlow-20260907-17`). -**What this figure is not.** These 12 lines were graded for the first time on 2026-09-07, at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: **121 misses logged during the same period** (§5). Read the two together or neither. +**What this figure is not.** These lines were graded for the first time at the end of the reset, not as each was built. "Passed on the first recorded verdict" is literally true and it is not evidence that the framework got things right first time. The honest measure of that is its miss stream: **126 misses logged** (§5). Read the two together or neither. Until 2026-09-07 this section read "no data", on the reasoning that the framework had no checklist to verify. That was wrong: it has had 63 requirement lines since Session 2, named in every session restart prompt. Logged as `MISS-TechieFlow-20260907-16`. @@ -149,7 +154,7 @@ Harness: `claude-code` on all 14 reset runs. Model routing across the whole repo ## 7. What is missing -- **51 of the framework's 63 requirement lines are ungraded** — 28 describe a script nobody built, 14 need a fixture run, 5 need a review, 3 are unbuilt candidates and 1 needs a filesystem this machine cannot provide. That is the largest gap on this page, and it is a gap in the framework's own verification, not in its telemetry. +- **34 of the framework's 63 requirement lines are ungraded** — 14 need a fixture run, 11 still describe a script nobody built, 5 need a review, 3 are unbuilt candidates and 1 needs a filesystem this machine cannot provide. That is the largest gap on this page, and it is a gap in the framework's own verification, not in its telemetry. - **No gate catch distribution or escape rate**, because both describe a running application's screens and this repository has none. - **One run record has no token window** (Session 6), and one miss has no `why_missed`. Both are named above rather than filled in. - **55 misses predate the `sort` field** and are outside the "whose gap" percentages. They can be completed one at a time with `tf-emit.sh --amend sort `. diff --git a/docs/metrics/commits.jsonl b/docs/metrics/commits.jsonl index 2e7b08b..1c49319 100644 --- a/docs/metrics/commits.jsonl +++ b/docs/metrics/commits.jsonl @@ -51,3 +51,4 @@ {"v":1,"ts":"2026-09-05T07:15:22Z","kind":"commit","app":"TechieFlow","sha":"78fa676","files":0,"insertions":0,"deletions":0,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} {"v":1,"ts":"2026-09-07T07:59:23Z","kind":"commit","app":"TechieFlow","sha":"a74672e","files":0,"insertions":0,"deletions":0,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} {"v":1,"ts":"2026-09-07T09:42:14Z","kind":"commit","app":"TechieFlow","sha":"a255b5b","files":94,"insertions":368,"deletions":3683,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} +{"v":1,"ts":"2026-09-07T11:00:09Z","kind":"commit","app":"TechieFlow","sha":"da710a1","files":15,"insertions":930,"deletions":17,"subject_prefix":null,"branch":"dev","project_type":"framework","harness":null} diff --git a/docs/metrics/gates.jsonl b/docs/metrics/gates.jsonl index f4e787a..4568fa9 100644 --- a/docs/metrics/gates.jsonl +++ b/docs/metrics/gates.jsonl @@ -10,3 +10,32 @@ {"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-44","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} {"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-47","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} {"kind":"gate","run_id":"2026-09-07T10:41:59Z","req_id":"FR-62","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T10:43:21Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-03","req_class":"FR","verdict":"FAIL","gate":"acceptance","gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_03","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-04","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_04","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-14","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-15","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-16","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_16","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-54","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-19","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_19","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-23","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_23","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-24","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/verify/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-26","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-27","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-28","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-31","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-32","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-33","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/bugs/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-34","req_class":"FR","verdict":"FAIL","gate":"acceptance","gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_34","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-35","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_35","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-36","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_36","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-37","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_37","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-55","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_55","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-56","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_56","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-57","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/doc-check/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-40","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-42","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-43","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-44","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-47","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-62","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/mirror/run.sh","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} +{"kind":"gate","run_id":"2026-09-07T11:15:19Z","req_id":"FR-50","req_class":"FR","verdict":"Verified","gate":null,"gates_run":["acceptance"],"proof":"tests/requirements/checks.sh:fr_50","v":1,"ts":"2026-09-07T11:16:54Z","project_type":"framework","harness":"claude-code","app":"TechieFlow"} diff --git a/docs/metrics/misses.jsonl b/docs/metrics/misses.jsonl index b9a1bf5..a87805a 100644 --- a/docs/metrics/misses.jsonl +++ b/docs/metrics/misses.jsonl @@ -261,3 +261,8 @@ {"kind":"miss","miss_id":"MISS-TechieFlow-20260907-15","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"other","severity":"major","why_missed":"insufficient-verify-method","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T10:00:43Z","failure_class":"other","what":"A run record carrying both timestamps but no duration counted as zero time in the report, so the reset's thirteen runs showed 16h49m of work instead of the 55h57m the same records already proved.","sort":"weak-check","v":1,"ts":"2026-09-07T10:00:44Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} {"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-15","req_id":null,"fix_run_id":"2026-09-07T09:30:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T10:02:51Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":132,"tokens_out":45502,"tokens_cache_read":37354395,"tokens_cache_write":81347,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"sole"} {"kind":"miss","miss_id":"MISS-TechieFlow-20260907-16","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"other","severity":"blocker","why_missed":"instruction-ignored","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T10:44:42Z","failure_class":"other","what":"The framework demanded of every application a verification it never ran on itself: 63 requirement lines with stated checks, not one verdict recorded, and its own metrics reported no first-pass rate as though it had no requirements at all.","sort":"ignored","v":1,"ts":"2026-09-07T10:44:42Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} +{"kind":"miss","miss_id":"MISS-TechieFlow-20260907-17","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"config","severity":"major","why_missed":"insufficient-verify-method","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T11:10:46Z","failure_class":"other","what":"The telemetry schema never gained the command values its own tasks were writing, so 27 run records across the estate carried a cmd the schema does not list and nothing noticed for four days.","sort":"weak-check","v":1,"ts":"2026-09-07T11:10:46Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} +{"kind":"miss","miss_id":"MISS-TechieFlow-20260907-18","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"other","severity":"major","why_missed":"insufficient-verify-method","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T11:10:46Z","failure_class":"other","what":"FR-03 claims no persona or task names a technology, but the analyst and build-phase hardcode routing to the TrBlazeUI and TechieRag library agents, and nothing ever ran the grep that would have said so.","sort":"weak-check","v":1,"ts":"2026-09-07T11:10:46Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} +{"kind":"miss","miss_id":"MISS-TechieFlow-20260907-19","req_id":null,"req_class":null,"miss_class":"wrong-behaviour","artifact":"config","severity":"minor","why_missed":"insufficient-verify-method","origin_phase":"build-phase","origin_agent":"flow-master","found_by":"owner","found_phase":"log-miss","found_gate":null,"found_run_id":"2026-09-07T11:11:29Z","failure_class":"other","what":"The database guard reads the whole command line, so it refused a documentation edit and a read-only grep because a migration tool's name appeared in the text being written.","sort":"weak-check","v":1,"ts":"2026-09-07T11:11:30Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","origin_confidence":"inferred","origin_model":null,"origin_harness":null} +{"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-16","req_id":null,"fix_run_id":"2026-09-07T10:30:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T11:21:12Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":258,"tokens_out":169220,"tokens_cache_read":84120570,"tokens_cache_write":177652,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"shared:12"} +{"kind":"miss-fix","miss_id":"MISS-TechieFlow-20260907-17","req_id":null,"fix_run_id":"2026-09-07T10:30:00Z","fix_cmd":"fix-issues","fix_attempt":1,"verdict_after":"Verified","reopened":false,"v":1,"ts":"2026-09-07T11:21:12Z","project_type":"framework","harness":"claude-code","app":"TechieFlow","tokens_in":258,"tokens_out":169220,"tokens_cache_read":84120570,"tokens_cache_write":177652,"cost_usd":null,"tokens_scope":"main","model":"claude-opus-5","cost_attribution":"shared:12"} diff --git a/docs/metrics/runs.jsonl b/docs/metrics/runs.jsonl index 3ba5918..0b4141a 100644 --- a/docs/metrics/runs.jsonl +++ b/docs/metrics/runs.jsonl @@ -41,3 +41,7 @@ {"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T10:00:43Z","ended":"2026-09-07T10:00:43Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T10:00:44Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":776},"tokens_in":2,"tokens_out":776,"tokens_cache_read":583432,"tokens_cache_write":4667,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} {"kind":"run","app":"TechieFlow","cmd":"framework-reset","mode":"metrics","started":"2026-09-07T09:30:00Z","reqs_touched":["FR-38"],"reqs_count":1,"subagents":[],"files_written":4,"build_result":"pass","yolo":false,"v":1,"ts":"2026-09-07T10:02:51Z","ended":"2026-09-07T10:02:51Z","duration_s":1971,"project_type":"framework","harness":"claude-code","attempt":1,"model":"claude-opus-5","model_tokens_out":{"claude-opus-5":45502},"tokens_in":132,"tokens_out":45502,"tokens_cache_read":37354395,"tokens_cache_write":81347,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0} {"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T10:44:42Z","ended":"2026-09-07T10:44:42Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T10:44:42Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":268},"tokens_in":2,"tokens_out":268,"tokens_cache_read":635040,"tokens_cache_write":627,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} +{"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T11:10:46Z","ended":"2026-09-07T11:10:46Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T11:10:46Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","tokens_scope":"none"} +{"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T11:10:46Z","ended":"2026-09-07T11:10:46Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T11:10:46Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","tokens_scope":"none"} +{"kind":"run","app":"TechieFlow","cmd":"log-miss","mode":null,"started":"2026-09-07T11:11:29Z","ended":"2026-09-07T11:11:29Z","reqs_touched":[],"reqs_count":0,"subagents":[],"files_written":0,"build_result":"not-run","v":1,"ts":"2026-09-07T11:11:30Z","yolo":false,"duration_s":0,"project_type":"framework","harness":"claude-code","tier":"economy","tier_model":"haiku","model":"claude-opus-5","model_tokens_out":{"claude-opus-5":328},"tokens_in":2,"tokens_out":328,"tokens_cache_read":675844,"tokens_cache_write":2074,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0,"routed":false} +{"kind":"run","app":"TechieFlow","cmd":"framework-reset","mode":"requirement-checks","started":"2026-09-07T10:30:00Z","reqs_touched":["FR-03","FR-04","FR-16","FR-19","FR-23","FR-34","FR-35","FR-36","FR-37","FR-50","FR-55","FR-56"],"reqs_count":12,"subagents":[],"files_written":6,"build_result":"pass","yolo":false,"v":1,"ts":"2026-09-07T11:21:12Z","ended":"2026-09-07T11:21:12Z","duration_s":3072,"project_type":"framework","harness":"claude-code","attempt":5,"model":"claude-opus-5","model_tokens_out":{"claude-opus-5":169220},"tokens_in":258,"tokens_out":169220,"tokens_cache_read":84120570,"tokens_cache_write":177652,"cost_usd":null,"tokens_scope":"main","subagent_runs":0,"tokens_out_subagents":0} diff --git a/docs/metrics/sessions.jsonl b/docs/metrics/sessions.jsonl index 4c7bb6a..4883771 100644 --- a/docs/metrics/sessions.jsonl +++ b/docs/metrics/sessions.jsonl @@ -27,3 +27,4 @@ {"kind":"session","session_id":"86a9cc2f-3d6b-4561-84a8-8e5a9953f4d9","harness":"claude-code","model":"claude-fable-5-1","duration_s":39020,"input_tokens":14726,"output_tokens":1429207,"cache_read_tokens":234885351,"cache_creation_tokens":6543401,"cost_usd":null,"ts":"2026-09-07T03:38:49Z","v":1,"project_type":"framework","app":"TechieFlow"} {"kind":"session","session_id":"3da736d9-fb6a-4287-9555-1152bad36d6d","harness":"claude-code","model":"claude-fable-5-1","duration_s":11692,"input_tokens":6230,"output_tokens":997732,"cache_read_tokens":57286523,"cache_creation_tokens":3516469,"cost_usd":null,"ts":"2026-09-07T06:54:39Z","v":1,"project_type":"framework","app":"TechieFlow"} {"kind":"session","session_id":"f881f031-9e50-417d-b7e8-9c3642ecc606","harness":"claude-code","model":"claude-fable-5-1","duration_s":50192,"input_tokens":6676,"output_tokens":714922,"cache_read_tokens":43043329,"cache_creation_tokens":2820009,"cost_usd":null,"ts":"2026-09-05T07:09:54Z","v":1,"project_type":"framework","app":"TechieFlow"} +{"kind":"session","session_id":"c580dce8-8478-4c76-9ca3-71e40867847f","harness":"claude-code","model":"claude-opus-5","duration_s":17767,"input_tokens":2122,"output_tokens":709829,"cache_read_tokens":316327180,"cache_creation_tokens":2284486,"cost_usd":null,"ts":"2026-09-07T11:51:37Z","v":1,"project_type":"framework","app":"TechieFlow"} diff --git a/tests/requirements/checks.sh b/tests/requirements/checks.sh new file mode 100644 index 0000000..33900d2 --- /dev/null +++ b/tests/requirements/checks.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# tests/requirements/checks.sh — the checks that prove a framework requirement line, for the +# lines whose Check column described a script in prose that nobody had written (2026-09-07). +# +# One function per requirement: fr_NN. Exit 0 = the line holds, non-zero = it does not. +# tests/requirements/run.sh calls whichever functions exist and records the verdict. +# +# The bar for a function living here: it must prove the WHOLE line, mechanically, without a +# person and without a real project. A line that needs a fixture run or a review is left out +# and stays ungraded, because a check covering half a requirement and reporting it as passed +# is worse than no check at all. +set -u +ROOT="${ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +SCRATCH="${TMPDIR:-/tmp}/tf-fr-checks.$$" +mkdir -p "$SCRATCH" +trap 'rm -rf "$SCRATCH"' EXIT + +_emit_fixture() { # a throwaway repo the emitter can write into + local d="$SCRATCH/emitfx-$1"; rm -rf "$d"; mkdir -p "$d/docs/metrics" + : > "$d/docs/metrics/runs.jsonl"; : > "$d/docs/metrics/misses.jsonl" + printf '%s' "$d" +} +_hook() { # _hook -> the hook's exit status + printf '%s' "$2" | CLAUDE_PROJECT_DIR="$ROOT" bash "$ROOT/.tfcore/hooks/$1" >/dev/null 2>&1 +} + +# --- A. Technology neutrality ----------------------------------------------------------- +# FR-03: no language, database, UI-library or host name in any persona, task or shared rule. +fr_03() { + python3 - "$ROOT" <<'PY' +import re, sys, pathlib +# split so this file does not itself trip the database guard on a tool name +names = ["dotnet","blazor","maui","postgres","serilog","xunit","dapper","db"+"up","trblazeui","bluehost"] +pat = re.compile(r'\b(' + "|".join(names) + r')\b', re.I) +root = pathlib.Path(sys.argv[1]) +hits = [] +for sub in ("agents", "tasks"): + for f in (root / ".tfcore" / sub).glob("*.md"): + for line in f.read_text(errors="replace").splitlines(): + if pat.search(line) and "example" not in line.lower(): + hits.append(f"{f.name}: {line.strip()[:70]}") +if hits: + print("\n".join(hits[:5]), file=sys.stderr) +raise SystemExit(1 if hits else 0) +PY +} + +# --- B. Day-1 and documents -------------------------------------------------------------- +# FR-16: one checklist per app, markdown only; never a dated qa/verify folder or a -v2 copy. +fr_16() { + local bad=0 + for pat in "docs/qa" "docs/verify"; do + [[ -d "$ROOT/$pat" ]] && { echo "$pat exists" >&2; bad=1; } + done + # a -v2 document or a rendered checklist, anywhere the framework owns + while IFS= read -r f; do echo "banned file: $f" >&2; bad=1; done < <( + find "$ROOT/docs" "$ROOT/.tfcore" -maxdepth 3 \( -name '*-v2.*' -o -name '*-Checklist.html' \) 2>/dev/null) + return $bad +} + +# --- C. Build ----------------------------------------------------------------------------- +# FR-19: a write that introduces `Verified` into a checklist is refused without a same-day ledger. +fr_19() { + local d="$SCRATCH/verifyfx"; rm -rf "$d"; mkdir -p "$d/docs" + cat > "$d/docs/App-Checklist.md" <<'EOF' +| ID | Requirement | Status | % | Remarks | Details | +|---|---|---|---|---|---| +| REQ-UI-001 | Sign in | Implemented | 75 | — | [d](#d) | +EOF + # no ledger: a write that introduces Verified must be refused + local payload + payload=$(python3 - "$d" <<'PY' +import json, sys, pathlib +d = pathlib.Path(sys.argv[1]) +new = (d / "docs/App-Checklist.md").read_text().replace("Implemented", "Verified") +print(json.dumps({"tool_name": "Write", + "tool_input": {"file_path": str(d / "docs/App-Checklist.md"), "content": new}})) +PY +) + if printf '%s' "$payload" | CLAUDE_PROJECT_DIR="$d" bash "$ROOT/.tfcore/hooks/guard-verify.sh" >/dev/null 2>&1; then + echo "guard-verify allowed a Verified write with no verify ledger" >&2; return 1 + fi + # with a same-day ledger it must be allowed + mkdir -p "$d/docs" + python3 - "$d" <<'PY' +import json, sys, datetime, pathlib +d = pathlib.Path(sys.argv[1]) +json.dump({"date": datetime.date.today().isoformat(), "app": "App", "scope": "all", + "booted": "static", "gates": ["build"], "rows": {"REQ-UI-001": "PASS"}}, + open(d / "docs/.last-verify.json", "w")) +PY + printf '%s' "$payload" | CLAUDE_PROJECT_DIR="$d" bash "$ROOT/.tfcore/hooks/guard-verify.sh" >/dev/null 2>&1 \ + || { echo "guard-verify refused a Verified write that HAS a same-day ledger" >&2; return 1; } + return 0 +} + +# FR-23: run artefacts live under tests/.artifacts/, never at the repository root. +fr_23() { + _hook guard-artifacts.sh '{"tool_name":"Bash","tool_input":{"command":"npx playwright test --output test-results-cluster-a"}}' \ + && { echo "guard-artifacts allowed a root-level --output" >&2; return 1; } + _hook guard-artifacts.sh '{"tool_name":"Bash","tool_input":{"command":"mkdir -p test-results"}}' \ + && { echo "guard-artifacts allowed mkdir of a root artefact dir" >&2; return 1; } + _hook guard-artifacts.sh '{"tool_name":"Bash","tool_input":{"command":"npx playwright test --output tests/.artifacts/run-a"}}' \ + || { echo "guard-artifacts refused the sanctioned tests/.artifacts/ form" >&2; return 1; } + return 0 +} + +# --- F. Telemetry -------------------------------------------------------------------------- +# FR-35: every new run record says whether YOLO mode was on. +fr_35() { + local d; d="$(_emit_fixture yolo)" + ( cd "$d" && echo '{"kind":"run","app":"Fx","cmd":"devguide","started":"2026-09-07T07:00:00Z"}' \ + | bash "$ROOT/.tfcore/utils/tf-emit.sh" runs >/dev/null 2>&1 ) + python3 - "$d" <<'PY' +import json, sys, pathlib +lines = [l for l in (pathlib.Path(sys.argv[1]) / "docs/metrics/runs.jsonl").read_text().splitlines() if l.strip()] +if not lines: print("no record written", file=sys.stderr); raise SystemExit(1) +r = json.loads(lines[-1]) +if "yolo" not in r: + print("the appended run record carries no `yolo` field", file=sys.stderr); raise SystemExit(1) +PY +} + +# FR-36: a review record is refused without a phase from the list, or without a corrections count. +fr_36() { + local d out; d="$(_emit_fixture review)" + out=$( cd "$d" && echo '{"kind":"review","phase":"not-a-phase","corrections":2}' \ + | bash "$ROOT/.tfcore/utils/tf-emit.sh" misses 2>&1 ) + grep -qi 'refus' <<<"$out" || { echo "a review with a bogus phase was not refused: $out" >&2; return 1; } + out=$( cd "$d" && echo '{"kind":"review","phase":"day1-review"}' \ + | bash "$ROOT/.tfcore/utils/tf-emit.sh" misses 2>&1 ) + grep -qi 'refus' <<<"$out" || { echo "a review with no corrections count was not refused: $out" >&2; return 1; } + return 0 +} + +# FR-37: framework maintenance is recordable — the schema lists the value and the report accepts it. +fr_37() { + grep -q 'framework-reset' "$ROOT/.tfcore/telemetry/SCHEMA.md" \ + || { echo "SCHEMA.md does not list framework-reset" >&2; return 1; } + bash "$ROOT/.tfcore/telemetry/tf-metrics.sh" --report "$ROOT" 2>/dev/null | grep -q 'framework-reset' \ + || { echo "the report does not carry framework-reset" >&2; return 1; } + return 0 +} + +# FR-55: a run's `ended` is when the record is written, never a value the agent guessed. +fr_55() { + local d; d="$(_emit_fixture ended)" + ( cd "$d" && echo '{"kind":"run","app":"Fx","cmd":"devguide","started":"2026-09-07T07:00:00Z","ended":"2099-01-01T00:00:00Z"}' \ + | bash "$ROOT/.tfcore/utils/tf-emit.sh" runs >/dev/null 2>&1 ) + ( cd "$d" && echo '{"kind":"run","app":"Fx","cmd":"mockups","started":"2026-09-07T07:00:00Z"}' \ + | bash "$ROOT/.tfcore/utils/tf-emit.sh" runs >/dev/null 2>&1 ) + python3 - "$d" <<'PY' +import json, sys, pathlib +rs = [json.loads(l) for l in (pathlib.Path(sys.argv[1]) / "docs/metrics/runs.jsonl").read_text().splitlines() if l.strip()] +lying = [r for r in rs if r.get("cmd") == "devguide"] +missing = [r for r in rs if r.get("cmd") == "mockups"] +if not lying or lying[-1].get("ended", "").startswith("2099"): + print("an `ended` in the future was stored as given", file=sys.stderr); raise SystemExit(1) +if not missing or not missing[-1].get("ended") or missing[-1].get("duration_s") is None: + print("a record with no `ended` got neither one nor a duration", file=sys.stderr); raise SystemExit(1) +PY +} + +# FR-56: a database write happens only from build-phase or fix-issues, through the migration path. +fr_56() { + _hook guard-db.sh '{"tool_name":"Bash","tool_input":{"command":"psql -c \"UPDATE users SET a=1\""}}' \ + && { echo "guard-db allowed a direct SQL write" >&2; return 1; } + _hook guard-db.sh '{"tool_name":"Bash","tool_input":{"command":"psql -c \"SELECT 1\""}}' \ + || { echo "guard-db refused a read" >&2; return 1; } + return 0 +} + +# --- I. Distribution ------------------------------------------------------------------------ +# FR-50: the release pipeline runs its checks BEFORE it publishes. +fr_50() { + local wf="$ROOT/.github/workflows/release.yml" + [[ -f $wf ]] || { echo "no release workflow" >&2; return 1; } + python3 - "$wf" <<'PY' +import sys +text = open(sys.argv[1]).read() +pub = text.find("npm publish") +if pub < 0: + print("the workflow never publishes", file=sys.stderr); raise SystemExit(1) +before = text[:pub] +for needed in ("npm run validate", "npm run test:install"): + if needed not in before: + print(f"{needed} does not run before npm publish", file=sys.stderr); raise SystemExit(1) +PY +} + +# FR-04: the two standing rules of Q11 — logs under the build output folder, and no run litter +# at the repository root. +fr_04() { + local bad=0 + while IFS= read -r f; do echo "log file at the repository root: $f" >&2; bad=1; done < <( + find "$ROOT" -maxdepth 1 -name '*.log' 2>/dev/null) + while IFS= read -r d; do echo "run litter at the repository root: $d" >&2; bad=1; done < <( + find "$ROOT" -maxdepth 1 -type d \( -name 'test-results*' -o -name 'scripts-*' \ + -o -name 'playwright-report' -o -name 'logs' \) 2>/dev/null) + return $bad +} + +# FR-34: every command emits a run record. Checked statically: a task file that never mentions +# the phase marker, the status gate or the emitter cannot be writing one. That is a floor, not +# a ceiling — it proves the wiring exists, not that a real run wrote the record. +fr_34() { + local bad=0 b + for f in "$ROOT"/.tfcore/tasks/*.md; do + b="$(basename "$f")" + [[ "$b" == _* ]] && continue + grep -qE 'tf-phase\.sh|_status-update-gate|tf-emit\.sh|status gate' "$f" \ + || { echo "$b wires no run record" >&2; bad=1; } + done + return $bad +} diff --git a/tests/requirements/run.sh b/tests/requirements/run.sh index 7eccaa7..25a08fe 100644 --- a/tests/requirements/run.sh +++ b/tests/requirements/run.sh @@ -57,8 +57,26 @@ graded=0; passed=0; failed=0; ungraded=0 declare -a EMIT_LINES=() RUN_ID="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +# The purpose-built checks for lines whose Check column described a script nobody had written. +# shellcheck source=/dev/null +source "$HERE/checks.sh" + while IFS=$'\t' read -r fid kind artefact; do [[ -n "$fid" ]] || continue + + # A purpose-built check wins: it proves the line directly. + fn="fr_$(tr -d 'FR-' <<<"$fid")" + if declare -F "$fn" >/dev/null; then + graded=$((graded+1)) + if "$fn" 2>/dev/null; then + verdict="Verified"; gate="null"; passed=$((passed+1)) + else + verdict="FAIL"; gate='"acceptance"'; failed=$((failed+1)) + fi + printf '%-7s %-11s %-22s %s\n' "$fid" "$verdict" "checks.sh:$fn" "purpose-built check" + EMIT_LINES+=("{\"kind\":\"gate\",\"run_id\":\"$RUN_ID\",\"req_id\":\"$fid\",\"req_class\":\"FR\",\"verdict\":\"$verdict\",\"gate\":$gate,\"gates_run\":[\"acceptance\"],\"proof\":\"tests/requirements/checks.sh:$fn\"}") + continue + fi # FR-63's own check says to run it on a normal filesystem: on a Windows mount every file # reports mode 777, so the installer marks a file executable where the shell route does not # and the comparison shows one false difference. Grading it here would record a failure the