From fa6f0361a1058fd4a9aad9c7dde4adec6efce0a8 Mon Sep 17 00:00:00 2001 From: jay23606 <4602056+jay23606@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:59:17 -0400 Subject: [PATCH 1/4] performance: keep results measured before a history reset out of the series persist_results appended any row whose commit and metric were absent from the destination CSV. A reset removes those identities, so a result measured before a reset commit was inserted after it and counted toward the ten-point warmup. A delayed dev run could therefore gate a metric that had only nine post-reset measurements. Before appending, persist now asks git which dimensions lost rows between each result's commit and HEAD, and skips those rows. A result whose commit is not an ancestor of HEAD fails the persist instead of being guessed. Unaffected metrics still persist. The CLI checks whenever its history directory is inside a git checkout, which the persist job always is. Adds a regression test in which a result from before a reset arrives after nine post-reset rows; the series stays at nine rows and the unaffected throughput row still persists. Documents the rule next to the reset contract. Fixes #511 (performance history reset race) Co-Authored-By: Claude Haiku 5.5 Signed-off-by: jay23606 <4602056+jay23606@users.noreply.github.com> --- doc/benchmarks.md | 2 + scripts/nvx_tools/performance.py | 116 ++++++++++++++++++++++++++++++- scripts/test_performance.py | 70 +++++++++++++++++++ 3 files changed, 187 insertions(+), 1 deletion(-) diff --git a/doc/benchmarks.md b/doc/benchmarks.md index 447c0887d..ca92772f6 100644 --- a/doc/benchmarks.md +++ b/doc/benchmarks.md @@ -736,6 +736,8 @@ host doesn't record, such as a performance tier or bursting setting, reset at le affected metrics by removing them from the series' history files, such as `data/windows-whp-virtual-machine-microvm-v2-1vcpu.csv`, so that the gate restarts their ten-point warmup instead of comparing the new storage with the old baseline. +Results measured before such a reset commit are not persisted into the reset +series, so a `dev` run that finishes after the reset cannot refill it. The regression gate compares the target p50 with the median of the latest 10 p50 values on the pull request's base branch and requires all 10 diff --git a/scripts/nvx_tools/performance.py b/scripts/nvx_tools/performance.py index 23785bbea..931f6b8c4 100644 --- a/scripts/nvx_tools/performance.py +++ b/scripts/nvx_tools/performance.py @@ -9,9 +9,10 @@ import os import re import statistics +import subprocess import sys import tempfile -from collections import defaultdict, deque +from collections import Counter, defaultdict, deque from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass, replace from pathlib import Path @@ -2248,11 +2249,121 @@ def _verify_result_hosts( ) +def _git(checkout: Path, *args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(checkout), *args], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise PerformanceError( + f"git {' '.join(args)} failed: {completed.stderr.strip()}" + ) + return completed.stdout + + +def _committed_results(top: Path, revision: str, relative: str) -> list[Result]: + shown = subprocess.run( + ["git", "-C", str(top), "show", f"{revision}:{relative}"], + check=False, + capture_output=True, + ) + if shown.returncode != 0: + return [] + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "history.csv" + path.write_bytes(shown.stdout) + return read_results(path) + + +def _reset_dimensions_since( + git_checkout: Path, + history_path: Path, + commit: str, +) -> set[tuple[str, int, int, str]]: + """Return the dimensions whose committed history lost rows after ``commit``. + + A result measured at ``commit`` predates any later commit that removes the + dimension's rows, so it must not be appended onto the reset series. + """ + top = Path(_git(git_checkout, "rev-parse", "--show-toplevel").strip()).resolve() + try: + relative = history_path.resolve().relative_to(top).as_posix() + except ValueError as error: + raise PerformanceError(f"{history_path} is outside the git checkout") from error + ancestor = subprocess.run( + ["git", "-C", str(top), "merge-base", "--is-ancestor", commit, "HEAD"], + check=False, + capture_output=True, + ) + if ancestor.returncode != 0: + raise PerformanceError( + f"result commit {commit} is not an ancestor of the target history" + ) + later = _git( + top, + "log", + "--reverse", + "--format=%H", + f"{commit}..HEAD", + "--", + relative, + ).split() + previous = Counter( + _dimension_key(result, "") + for result in _committed_results(top, commit, relative) + ) + resets: set[tuple[str, int, int, str]] = set() + for revision in later: + current = Counter( + _dimension_key(result, "") + for result in _committed_results(top, revision, relative) + ) + resets.update(key for key, count in previous.items() if current[key] < count) + previous = current + return resets + + +def _without_reset_rows( + git_checkout: Path, + history_path: Path, + results: Sequence[Result], +) -> list[Result]: + resets: dict[str, set[tuple[str, int, int, str]]] = {} + kept: list[Result] = [] + for result in results: + if result.commit not in resets: + resets[result.commit] = _reset_dimensions_since( + git_checkout, history_path, result.commit + ) + if _dimension_key(result, "") in resets[result.commit]: + print( + f"Skipped {result.metric} measured at {result.commit[:12]}: " + f"{history_path.name} was reset after that commit" + ) + continue + kept.append(result) + return kept + + +def _git_checkout_of(path: Path) -> Path | None: + """Return ``path`` when it lies inside a git checkout, which CI always does.""" + probe = subprocess.run( + ["git", "-C", str(path), "rev-parse", "--is-inside-work-tree"], + check=False, + capture_output=True, + text=True, + ) + return path if probe.returncode == 0 and probe.stdout.strip() == "true" else None + + def persist_results( source_dir: Path, history_dir: Path, excluded_metrics: Iterable[str] = (), require_host: bool = False, + git_checkout: Path | None = None, ) -> None: source_files = sorted(source_dir.glob("*.csv")) if not source_files: @@ -2306,6 +2417,8 @@ def persist_results( for result in current if _result_identity(result) not in existing_keys ] + if git_checkout is not None: + new_results = _without_reset_rows(git_checkout, history_path, new_results) if not new_results: print(f"No new performance rows to persist for {source_path.name}") continue @@ -2916,6 +3029,7 @@ def command_performance(args: argparse.Namespace) -> int: args.history_dir, args.exclude_metric, require_host=args.require_host, + git_checkout=_git_checkout_of(args.history_dir), ) return 0 except (PerformanceError, OSError, csv.Error) as error: diff --git a/scripts/test_performance.py b/scripts/test_performance.py index d1dd16bcd..4c307c99d 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -7,6 +7,7 @@ import json import re import statistics +import subprocess import sys import tempfile import unittest @@ -2389,6 +2390,75 @@ def test_rejects_lifecycle_workload_processor_mismatch(self): lifecycle_input=lifecycle_path, ) + def test_persist_drops_rows_measured_before_a_later_history_reset(self): + # A row measured before a commit that reset its metric must not refill the reset + # series, or nine post-reset rows would already gate the metric. Unaffected metrics + # measured at the same commit still persist. + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) + + def git(*args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def commit(message: str) -> str: + git("add", "data") + git("commit", "-q", "--allow-empty", "-m", message) + return git("rev-parse", "HEAD") + + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "test") + history = repo / "data" + source = repo / "source" + history_file = history / "linux-kvm.csv" + source_file = source / "linux-kvm.csv" + performance.write_results( + history_file, + [ + performance.Result("seed", "latency", "ms", "lower", 10.0), + performance.Result("seed", "throughput", "MB/s", "higher", 20.0), + ], + ) + measured = commit("seed") + + # Commit B resets only the latency metric; throughput keeps its history. + performance.write_results( + history_file, + [performance.Result("seed", "throughput", "MB/s", "higher", 20.0)], + ) + commit("reset latency") + + for number in range(9): + sha = commit(f"post-reset {number}") + performance.write_results( + source_file, + [performance.Result(sha, "latency", "ms", "lower", 100.0)], + ) + performance.persist_results(source, history, git_checkout=repo) + + # The measurement of A arrives after the reset. It must not refill latency. + performance.write_results( + source_file, + [ + performance.Result(measured, "latency", "ms", "lower", 10.0), + performance.Result(measured, "throughput", "MB/s", "higher", 20.0), + ], + ) + performance.persist_results(source, history, git_checkout=repo) + + rows = performance.read_results(history_file) + latency = [row for row in rows if row.metric == "latency"] + throughput = [row for row in rows if row.metric == "throughput"] + self.assertEqual(len(latency), 9) + self.assertNotIn(measured, {row.commit for row in latency}) + self.assertEqual({row.commit for row in throughput}, {"seed", measured}) + def test_persist_and_gate_isolate_processor_counts(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) From 6e9de334606e6862753eeecf924931004c5475c9 Mon Sep 17 00:00:00 2001 From: jay23606 <4602056+jay23606@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:23:30 -0400 Subject: [PATCH 2/4] performance: compare reset commits with their parents, keep legacy platform The reset check compared consecutive commits in log order. A side branch forked before the measurement can hold fewer rows without resetting them, so it was misread as a reset. Compare each commit with its own parents instead, and include merged history with --full-history. A legacy CSV takes its platform from its file name, so the committed blob keeps its basename when it is read back. Adds a regression test in which a side branch changes latency without throughput and is merged with A's throughput row intact. Before this change the check flagged throughput. Co-Authored-By: Claude Haiku 5.5 Signed-off-by: jay23606 <4602056+jay23606@users.noreply.github.com> --- scripts/nvx_tools/performance.py | 38 ++++++++++++-------- scripts/test_performance.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/scripts/nvx_tools/performance.py b/scripts/nvx_tools/performance.py index 931f6b8c4..dbbc5f219 100644 --- a/scripts/nvx_tools/performance.py +++ b/scripts/nvx_tools/performance.py @@ -2272,11 +2272,23 @@ def _committed_results(top: Path, revision: str, relative: str) -> list[Result]: if shown.returncode != 0: return [] with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "history.csv" + # Keep the basename: a legacy CSV takes its platform from the file name. + path = Path(temporary) / Path(relative).name path.write_bytes(shown.stdout) return read_results(path) +def _dimension_counts( + top: Path, + revision: str, + relative: str, +) -> Counter[tuple[str, int, int, str]]: + return Counter( + _dimension_key(result, "") + for result in _committed_results(top, revision, relative) + ) + + def _reset_dimensions_since( git_checkout: Path, history_path: Path, @@ -2301,27 +2313,25 @@ def _reset_dimensions_since( raise PerformanceError( f"result commit {commit} is not an ancestor of the target history" ) + # Compare each commit with its own parents: a side branch forked before + # ``commit`` can hold fewer rows than ``commit`` without having reset them. later = _git( top, "log", "--reverse", - "--format=%H", + "--full-history", + "--format=%H %P", f"{commit}..HEAD", "--", relative, - ).split() - previous = Counter( - _dimension_key(result, "") - for result in _committed_results(top, commit, relative) - ) + ).splitlines() resets: set[tuple[str, int, int, str]] = set() - for revision in later: - current = Counter( - _dimension_key(result, "") - for result in _committed_results(top, revision, relative) - ) - resets.update(key for key, count in previous.items() if current[key] < count) - previous = current + for line in later: + revision, *parents = line.split() + current = _dimension_counts(top, revision, relative) + for parent in parents: + before = _dimension_counts(top, parent, relative) + resets.update(key for key, count in before.items() if current[key] < count) return resets diff --git a/scripts/test_performance.py b/scripts/test_performance.py index 4c307c99d..c78f018e0 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -2390,6 +2390,65 @@ def test_rejects_lifecycle_workload_processor_mismatch(self): lifecycle_input=lifecycle_path, ) + def test_reset_check_ignores_side_branch_history_that_never_reset_a_row(self): + # A side branch forked before the measurement can hold fewer rows than the + # measurement without resetting them. Comparing it with the previous commit + # in log order would flag the throughput row, which it never removed. + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) + + def git(*args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "test") + history_file = repo / "data" / "linux-kvm.csv" + performance.write_results( + history_file, + [performance.Result("s0", "latency", "ms", "lower", 10.0)], + ) + git("add", "data") + git("commit", "-q", "-m", "seed") + main = git("rev-parse", "--abbrev-ref", "HEAD") + git("checkout", "-q", "-b", "side") + performance.write_results( + history_file, + [performance.Result("x", "latency", "ms", "lower", 11.0)], + ) + git("commit", "-q", "-am", "side change") + git("checkout", "-q", main) + performance.write_results( + history_file, + [ + performance.Result("s0", "latency", "ms", "lower", 10.0), + performance.Result("A", "throughput", "MB/s", "higher", 20.0), + ], + ) + git("commit", "-q", "-am", "measured") + measured = git("rev-parse", "HEAD") + # The merge keeps the side branch's latency change and A's throughput row. + git("merge", "-q", "--no-commit", "--no-ff", "-X", "ours", "side") + performance.write_results( + history_file, + [ + performance.Result("x", "latency", "ms", "lower", 11.0), + performance.Result("A", "throughput", "MB/s", "higher", 20.0), + ], + ) + git("commit", "-q", "-am", "merge side") + + self.assertEqual( + performance._reset_dimensions_since(repo, history_file, measured), + set(), + ) + def test_persist_drops_rows_measured_before_a_later_history_reset(self): # A row measured before a commit that reset its metric must not refill the reset # series, or nine post-reset rows would already gate the metric. Unaffected metrics From b4fe24ad33f4db7df330d39ca8b2e8cceca459e0 Mon Sep 17 00:00:00 2001 From: jay23606 <4602056+jay23606@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:46:26 -0400 Subject: [PATCH 3/4] performance: only compare reset merges with history on the measurement's line A branch forked before an earlier reset can keep rows that the target already reset before the measurement. Merging it drops those rows relative to that branch, which was misread as a reset after the measurement, and its valid result was discarded. Compare a commit with a parent only when the parent came before the measurement or was built on it. Adds a regression test for that merge. Co-Authored-By: Claude Haiku 5.5 Signed-off-by: jay23606 <4602056+jay23606@users.noreply.github.com> --- scripts/nvx_tools/performance.py | 25 +++++++++++++++ scripts/test_performance.py | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/scripts/nvx_tools/performance.py b/scripts/nvx_tools/performance.py index dbbc5f219..eea444e53 100644 --- a/scripts/nvx_tools/performance.py +++ b/scripts/nvx_tools/performance.py @@ -2330,11 +2330,36 @@ def _reset_dimensions_since( revision, *parents = line.split() current = _dimension_counts(top, revision, relative) for parent in parents: + # A branch that neither predates nor follows the measurement can drop rows + # that were already reset before it, so only the measurement's line counts. + if not _on_measurement_line(top, parent, commit): + continue before = _dimension_counts(top, parent, relative) resets.update(key for key, count in before.items() if current[key] < count) return resets +def _on_measurement_line(top: Path, revision: str, measured: str) -> bool: + """Return whether ``revision`` came before ``measured`` or was built on it.""" + for ancestor, descendant in ((revision, measured), (measured, revision)): + check = subprocess.run( + [ + "git", + "-C", + str(top), + "merge-base", + "--is-ancestor", + ancestor, + descendant, + ], + check=False, + capture_output=True, + ) + if check.returncode == 0: + return True + return False + + def _without_reset_rows( git_checkout: Path, history_path: Path, diff --git a/scripts/test_performance.py b/scripts/test_performance.py index c78f018e0..470153c3e 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -2449,6 +2449,60 @@ def git(*args: str) -> str: set(), ) + def test_reset_check_ignores_a_merged_branch_that_predates_an_earlier_reset(self): + # Dev resets latency before measurement A. A branch forked before that reset + # keeps the old latency rows and adds throughput; merging it into dev keeps the + # reset, so the merge drops latency relative to that branch. The reset preceded A, + # so A's latency row must not be discarded. + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) + + def git(*args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "test") + history_file = repo / "data" / "linux-kvm.csv" + performance.write_results( + history_file, + [performance.Result("s0", "latency", "ms", "lower", 10.0)], + ) + git("add", "data") + git("commit", "-q", "-m", "seed") + main = git("rev-parse", "--abbrev-ref", "HEAD") + git("checkout", "-q", "-b", "branch") + performance.write_results( + history_file, + [ + performance.Result("s0", "latency", "ms", "lower", 12.0), + performance.Result("b", "throughput", "MB/s", "higher", 5.0), + ], + ) + git("commit", "-q", "-am", "branch work") + git("checkout", "-q", main) + performance.write_results(history_file, []) + git("commit", "-q", "-am", "reset latency before the measurement") + git("commit", "-q", "--allow-empty", "-m", "measured") + measured = git("rev-parse", "HEAD") + git("merge", "-q", "--no-commit", "--no-ff", "-X", "ours", "branch") + performance.write_results( + history_file, + [performance.Result("b", "throughput", "MB/s", "higher", 5.0)], + ) + git("commit", "-q", "-am", "merge branch") + + self.assertEqual( + performance._reset_dimensions_since(repo, history_file, measured), + set(), + ) + def test_persist_drops_rows_measured_before_a_later_history_reset(self): # A row measured before a commit that reset its metric must not refill the reset # series, or nine post-reset rows would already gate the metric. Unaffected metrics From a3bb4a94e8d1ec11f3676064ad11d5445a105d64 Mon Sep 17 00:00:00 2001 From: jay23606 <4602056+jay23606@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:40:31 -0400 Subject: [PATCH 4/4] Address review feedback --- scripts/nvx_tools/performance.py | 36 ++++++---- scripts/test_performance.py | 117 +++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 15 deletions(-) diff --git a/scripts/nvx_tools/performance.py b/scripts/nvx_tools/performance.py index eea444e53..f4386812e 100644 --- a/scripts/nvx_tools/performance.py +++ b/scripts/nvx_tools/performance.py @@ -2320,6 +2320,7 @@ def _reset_dimensions_since( "log", "--reverse", "--full-history", + "--sparse", "--format=%H %P", f"{commit}..HEAD", "--", @@ -2328,6 +2329,10 @@ def _reset_dimensions_since( resets: set[tuple[str, int, int, str]] = set() for line in later: revision, *parents = line.split() + # Ignore commits on side branches that are not part of the history + # leading from the measurement to HEAD, even if later merged/reverted. + if not _on_measurement_line(top, revision, commit): + continue current = _dimension_counts(top, revision, relative) for parent in parents: # A branch that neither predates nor follows the measurement can drop rows @@ -2340,9 +2345,9 @@ def _reset_dimensions_since( def _on_measurement_line(top: Path, revision: str, measured: str) -> bool: - """Return whether ``revision`` came before ``measured`` or was built on it.""" - for ancestor, descendant in ((revision, measured), (measured, revision)): - check = subprocess.run( + """Return whether ``revision`` is on the path from ``measured`` to HEAD.""" + return all( + subprocess.run( [ "git", "-C", @@ -2354,10 +2359,10 @@ def _on_measurement_line(top: Path, revision: str, measured: str) -> bool: ], check=False, capture_output=True, - ) - if check.returncode == 0: - return True - return False + ).returncode + == 0 + for ancestor, descendant in ((measured, revision), (revision, "HEAD")) + ) def _without_reset_rows( @@ -2421,6 +2426,14 @@ def persist_results( current = [result for result in source_results if result.metric not in excluded] history_path = history_dir / source_path.name existing = read_results(history_path) if history_path.exists() else [] + existing_keys = {_result_identity(result) for result in existing} + current = [ + result + for result in current + if _result_identity(result) not in existing_keys + ] + if git_checkout is not None: + current = _without_reset_rows(git_checkout, history_path, current) metadata = { ( result.platform, @@ -2446,14 +2459,7 @@ def persist_results( f"{expected} -> {actual}" ) - existing_keys = {_result_identity(result) for result in existing} - new_results = [ - result - for result in current - if _result_identity(result) not in existing_keys - ] - if git_checkout is not None: - new_results = _without_reset_rows(git_checkout, history_path, new_results) + new_results = current if not new_results: print(f"No new performance rows to persist for {source_path.name}") continue diff --git a/scripts/test_performance.py b/scripts/test_performance.py index 470153c3e..d75c0ad3b 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -2572,6 +2572,123 @@ def commit(message: str) -> str: self.assertNotIn(measured, {row.commit for row in latency}) self.assertEqual({row.commit for row in throughput}, {"seed", measured}) + def test_persist_detects_merged_reset_and_ignores_reverted_side_reset(self): + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) + + def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def commit(message: str) -> str: + git("add", "data") + git("commit", "-q", "--allow-empty", "-m", message) + return git("rev-parse", "HEAD") + + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "test") + history_file = repo / "data" / "linux-kvm.csv" + original = [performance.Result("seed", "latency", "ms", "lower", 10.0)] + performance.write_results(history_file, original) + commit("seed") + main = git("rev-parse", "--abbrev-ref", "HEAD") + + # A side branch forked before the measurement resets and restores + # rows; its temporary reset is outside the measured ancestry. + git("checkout", "-q", "-b", "reverted") + unrelated = repo / "data" / "unrelated.txt" + unrelated.write_text("x", encoding="utf-8") + git("add", "data/unrelated.txt") + git("commit", "-q", "-m", "unrelated") + performance.write_results(history_file, []) + commit("temporary reset") + performance.write_results(history_file, original) + commit("restore rows") + git("checkout", "-q", main) + measured = commit("measurement") + git("merge", "-q", "--no-ff", "reverted", "-m", "merge restored branch") + + source = repo / "source" + performance.write_results( + source / "linux-kvm.csv", + [performance.Result(measured, "latency", "ms", "lower", 11.0)], + ) + performance.persist_results(source, repo / "data", git_checkout=repo) + rows = performance.read_results(history_file) + self.assertIn(measured, {row.commit for row in rows}) + + # A second branch performs a multi-commit reset. The integration + # merge must expose the loss even when git's path simplification + # would otherwise omit an intermediate commit. + git("checkout", "-q", "-b", "reset-branch") + performance.write_results(history_file, []) + commit("begin reset") + commit("continue reset") + git("checkout", "-q", main) + git("merge", "-q", "--no-ff", "reset-branch", "-m", "merge reset") + performance.write_results( + source / "linux-kvm.csv", + [performance.Result(measured, "latency", "ms", "lower", 12.0)], + ) + performance.persist_results(source, repo / "data", git_checkout=repo) + self.assertEqual(performance.read_results(history_file), []) + + def test_persist_filters_stale_metadata_before_validation(self): + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) + + def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def commit(message: str) -> str: + git("add", "data") + git("commit", "-q", "-m", message) + return git("rev-parse", "HEAD") + + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "test") + history_file = repo / "data" / "linux-kvm.csv" + performance.write_results( + history_file, + [ + performance.Result("seed", "latency", "ms", "lower", 10.0), + performance.Result("seed", "throughput", "MB/s", "higher", 20.0), + ], + ) + measured = commit("measurement") + performance.write_results( + history_file, + [ + performance.Result("seed", "throughput", "MB/s", "higher", 20.0), + ], + ) + commit("reset latency") + source = repo / "source" + performance.write_results( + source / "linux-kvm.csv", + [ + performance.Result(measured, "latency", "seconds", "higher", 1.0), + performance.Result(measured, "throughput", "MB/s", "higher", 21.0), + ], + ) + performance.persist_results(source, repo / "data", git_checkout=repo) + rows = performance.read_results(history_file) + self.assertEqual( + [(row.metric, row.commit) for row in rows], + [("throughput", "seed"), ("throughput", measured)], + ) + def test_persist_and_gate_isolate_processor_counts(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary)