test data repo

.github/scripts/perf_publish.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Append one master measurement to the perf history repo.
+# Needs PERF_DATA_REPO, PERF_DATA_KEY (deploy key with write access),
+# SHA and COMMITTED_AT in the environment.
+set -ueo pipefail
+
+results="$1"
+clone="$RUNNER_TEMP/perf-data"
+key=~/.ssh/perf_data
+
+mkdir -p ~/.ssh
+printf '%s\n' "$PERF_DATA_KEY" > "$key"
+chmod 600 "$key"
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2> /dev/null
+export GIT_SSH_COMMAND="ssh -i $key -o IdentitiesOnly=yes"
+
+git clone "git@github.com:$PERF_DATA_REPO.git" "$clone"
+cd "$clone"
+git config user.name dmd-perf-bot
+git config user.email dmd-perf-bot@users.noreply.github.com
+
+dest="data/${COMMITTED_AT:0:4}/${COMMITTED_AT:5:2}"
+mkdir -p "$dest"
+cp "$results" "$dest/$SHA.json"
+git add "$dest/$SHA.json"
+
+if git diff --cached --quiet; then
+    echo "$SHA already published"
+    exit 0
+fi
+
+git commit -m "$SHA"
+
+# Another master run may have published while we were measuring.
+for _ in 1 2 3; do
+    if git push; then
+        exit 0
+    fi
+    git pull --rebase
+done
+exit 1

.github/workflows/perf.yml

@@ -9,9 +9,10 @@ on:
   push:
     branches: [master]
 
+# Master runs queue instead of cancelling: every one of them is a data point.
 concurrency:
   group: perf-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -42,12 +43,24 @@ jobs:
         run: |
           set -uexo pipefail
           HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
-          git fetch --no-tags origin master
-          BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)"
-          echo "head=$HEAD_SHA"     >> "$GITHUB_OUTPUT"
-          echo "base=$BASE_SHA"     >> "$GITHUB_OUTPUT"
+          echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT"
           echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT"
 
+          # On master the merge-base of the pushed commit is the commit itself,
+          # so there is nothing to diff against: measure it on its own instead.
+          if [ "${{ github.event_name }}" = pull_request ]; then
+            git fetch --no-tags origin master
+            echo "base=$(git merge-base "$HEAD_SHA" origin/master)" >> "$GITHUB_OUTPUT"
+          else
+            BEFORE="${{ github.event.before }}"
+            # Unset after a force-push or on a brand new branch.
+            git rev-parse -q --verify "$BEFORE^{commit}" > /dev/null || BEFORE="$HEAD_SHA~"
+            echo "base=" >> "$GITHUB_OUTPUT"
+            echo "before=$BEFORE" >> "$GITHUB_OUTPUT"
+            echo "commits=$(git rev-list --count "$BEFORE..$HEAD_SHA")" >> "$GITHUB_OUTPUT"
+            echo "date=$(git show -s --format=%cI "$HEAD_SHA")" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Install prerequisites
         run: |
           sudo apt-get update
@@ -67,6 +80,7 @@ jobs:
           source ~/dlang/*/activate
           built=generated/$OS_NAME/release/$MODEL
           profile="$RUNNER_TEMP/pgo/merged.data"
+          BASE_SHA="${{ steps.refs.outputs.base }}"
           mkdir -p "$RUNNER_TEMP/pgo"
 
           # Check out a ref and build a plain dmd + phobos
@@ -88,32 +102,55 @@ jobs:
                 DFLAGS="-fprofile-instr-use=$profile -wi" -j$N --force )
           }
 
-          setup_ref "${{ steps.refs.outputs.base }}" base
           setup_ref "${{ steps.refs.outputs.head }}" head
+          # Train on base where there is one, so the profile cannot favour the PR's code.
+          if [ -n "$BASE_SHA" ]; then
+            setup_ref "$BASE_SHA" base
+            train=base
+          else
+            train=head
+          fi
 
           # Train PGO once, then reuse the profile for both refs.
-          ( cd "$RUNNER_TEMP/base/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force )
-          cp "$RUNNER_TEMP/base/dmd/$built/dmd_profdata/merged.data" "$profile"
-          build_with_profile base
+          ( cd "$RUNNER_TEMP/$train/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force )
+          cp "$RUNNER_TEMP/$train/dmd/$built/dmd_profdata/merged.data" "$profile"
           build_with_profile head
+          if [ -n "$BASE_SHA" ]; then
+            build_with_profile base
+          fi
           deactivate
 
       - name: Measure
         run: |
           set -uexo pipefail
           source ~/dlang/*/activate
           built=generated/$OS_NAME/release/$MODEL/dmd
-          cd tools/perfrunner
-          dub run -- \
-            --base-dmd "$RUNNER_TEMP/base/dmd/$built" \
-            --head-dmd "$RUNNER_TEMP/head/dmd/$built" \
-            --base-phobos "$RUNNER_TEMP/base/phobos" \
-            --head-phobos "$RUNNER_TEMP/head/phobos" \
-            --base-sha "${{ steps.refs.outputs.base }}" \
-            --head-sha "${{ steps.refs.outputs.head }}" \
-            --pr "${{ github.event.pull_request.number || 0 }}" \
-            --host-dmd "$HOST_DMD" \
+          BASE_SHA="${{ steps.refs.outputs.base }}"
+
+          args=(
+            --head-dmd "$RUNNER_TEMP/head/dmd/$built"
+            --head-phobos "$RUNNER_TEMP/head/phobos"
+            --head-sha "${{ steps.refs.outputs.head }}"
+            --host-dmd "$HOST_DMD"
             --out "$GITHUB_WORKSPACE/results.json"
+          )
+          if [ -n "$BASE_SHA" ]; then
+            args+=(
+              --base-dmd "$RUNNER_TEMP/base/dmd/$built"
+              --base-phobos "$RUNNER_TEMP/base/phobos"
+              --base-sha "$BASE_SHA"
+              --pr "${{ github.event.pull_request.number || 0 }}"
+            )
+          else
+            args+=(
+              --before "${{ steps.refs.outputs.before }}"
+              --commits "${{ steps.refs.outputs.commits }}"
+              --committed-at "${{ steps.refs.outputs.date }}"
+            )
+          fi
+
+          cd tools/perfrunner
+          dub run -- "${args[@]}"
 
       - name: Show results
         run: cat "$GITHUB_WORKSPACE/results.json"
@@ -125,3 +162,13 @@ jobs:
         with:
           name: perf-results
           path: results.json
+
+      # Skipped until PERF_DATA_REPO is configured, so this is safe to merge early.
+      - name: Publish to the history repo
+        if: github.event_name == 'push' && vars.PERF_DATA_REPO != ''
+        env:
+          PERF_DATA_REPO: ${{ vars.PERF_DATA_REPO }}
+          PERF_DATA_KEY: ${{ secrets.PERF_DATA_KEY }}
+          SHA: ${{ steps.refs.outputs.head }}
+          COMMITTED_AT: ${{ steps.refs.outputs.date }}
+        run: .github/scripts/perf_publish.sh "$GITHUB_WORKSPACE/results.json"

tools/perfrunner/source/app.d

@@ -6,7 +6,7 @@ import std.path : buildPath, dirName;
 import std.stdio : stderr, writeln;
 
 import metrics : measure, initials;
-import report : MetricResult, render, Report;
+import report : CommitRecord, MetricResult, render, renderCommit, Report;
 import vibed : describeFlags;
 
 enum workloads = buildPath(__FILE_FULL_PATH__.dirName.dirName, "workloads");
@@ -21,9 +21,10 @@ version (unittest) {} else
 int main(string[] args)
 {
     string baseDmd, headDmd, basePhobos, headPhobos, baseSha, headSha, hostDmd;
+    string before, committedAt;
     string os = "ubuntu-latest";
     string outPath = "results.json";
-    long pr;
+    long pr, commits = 1;
 
     auto help = getopt(args,
         "base-dmd", "path to the base (merge-base) dmd binary", &baseDmd,
@@ -33,31 +34,50 @@ int main(string[] args)
         "base-sha", "base commit sha (metadata)", &baseSha,
         "head-sha", "head commit sha (metadata)", &headSha,
         "pr",       "pull request number (metadata)", &pr,
+        "before",   "sha master pointed at before the push (metadata)", &before,
+        "commits",  "commits contained in the push (metadata)", &commits,
+        "committed-at", "commit timestamp (metadata)", &committedAt,
         "os",       "runner OS label (metadata)", &os,
         "host-dmd", "bootstrap dmd version (metadata)", &hostDmd,
         "out",      "where to write results.json", &outPath,
     );
 
     if (help.helpWanted)
     {
-        writeln("usage: perfrunner --base-dmd <path> --head-dmd <path> "
-            ~ "--base-phobos <dir> --head-phobos <dir> "
+        writeln("usage: perfrunner --head-dmd <path> --head-phobos <dir> "
+            ~ "[--base-dmd <path> --base-phobos <dir>] "
             ~ "[--base-sha <sha> --head-sha <sha> --pr <n>] --out results.json");
+        writeln("without a base the single commit is measured for the history repo");
         return 0;
     }
 
-    if (baseDmd.length == 0 || headDmd.length == 0
-        || basePhobos.length == 0 || headPhobos.length == 0)
+    if (headDmd.length == 0 || headPhobos.length == 0)
     {
-        stderr.writeln("error: --base-dmd, --head-dmd, --base-phobos and --head-phobos are required");
+        stderr.writeln("error: --head-dmd and --head-phobos are required");
+        return 2;
+    }
+
+    immutable diff = baseDmd.length != 0;
+    if (diff && basePhobos.length == 0)
+    {
+        stderr.writeln("error: --base-dmd needs --base-phobos");
         return 2;
     }
 
     auto tmp = buildPath(tempDir, "perfrunner");
     mkdirRecurse(tmp);
 
     // Resolved once so both refs compile vibe.d with the same flags.
-    auto vibedFlags = describeFlags(vibedDir, baseDmd);
+    auto vibedFlags = describeFlags(vibedDir, diff ? baseDmd : headDmd);
+
+    if (!diff)
+    {
+        auto m = measure(headDmd, workload, headPhobos, vibedRoot, vibedFlags, tmp, "head");
+        write(outPath, renderCommit(CommitRecord(headSha, committedAt, before, commits,
+            os, hostDmd, m.metrics, m.helloTrace, m.phobosTrace)));
+        writeln("wrote ", outPath);
+        return 0;
+    }
 
     auto base = measure(baseDmd, workload, basePhobos, vibedRoot, vibedFlags, tmp, "base");
     auto head = measure(headDmd, workload, headPhobos, vibedRoot, vibedFlags, tmp, "head");

tools/perfrunner/source/report.d

@@ -29,6 +29,20 @@ struct Report
     Trace phobosBase, phobosHead;
 }
 
+// One master commit, measured on its own. `before` and `commits` describe the
+// push it arrived in, since a rebase-merged PR lands as several commits at once.
+struct CommitRecord
+{
+    string sha;
+    string committedAt;
+    string before;
+    long commits;
+    string os;
+    string hostDmd;
+    long[string] metrics;
+    Trace hello, phobos;
+}
+
 // Serialise a report to the initial schema
 string render(Report rep)
 {
@@ -61,6 +75,29 @@ string render(Report rep)
     return root.toPrettyString();
 }
 
+// Serialise a single-commit measurement for the history repo
+string renderCommit(CommitRecord rec)
+{
+    JSONValue[string] metrics;
+    foreach (id, value; rec.metrics)
+        metrics[id] = JSONValue(value);
+
+    JSONValue root = [
+        "schema_version": JSONValue(3),
+        "commit":       JSONValue(rec.sha),
+        "committed_at": JSONValue(rec.committedAt),
+        "push":    JSONValue(["before": JSONValue(rec.before), "commits": JSONValue(rec.commits)]),
+        "runner":  JSONValue(["os": JSONValue(rec.os), "host_dmd": JSONValue(rec.hostDmd)]),
+        "metrics": JSONValue(metrics),
+        "time_trace": JSONValue([
+            "hello":  traceJson(rec.hello),
+            "phobos": traceJson(rec.phobos),
+        ]),
+    ];
+
+    return root.toPrettyString();
+}
+
 private JSONValue pair(long base, long head)
 {
     return JSONValue(["base": JSONValue(base), "head": JSONValue(head)]);
@@ -78,6 +115,18 @@ private JSONValue traceJson(Trace b, Trace h)
     ]);
 }
 
+private JSONValue traceJson(Trace t)
+{
+    JSONValue[string] phases;
+    foreach (id; phaseIds)
+        phases[id] = JSONValue(t.phase(id));
+
+    return JSONValue([
+        "total_us": JSONValue(t.total),
+        "phases":   JSONValue(phases),
+    ]);
+}
+
 unittest
 {
     auto rep = Report("base1", "merge-base", "head1", 7, "ubuntu-latest", "2.112.0",
@@ -97,3 +146,16 @@ unittest
     import std.math : isClose;
     assert(isClose(m["delta_pct"].floating, 1.0));
 }
+
+unittest
+{
+    auto rec = CommitRecord("head1", "2026-07-30T09:12:44Z", "before1", 3,
+        "ubuntu-latest", "ldc-1.42.0", ["compile_hello_debug_instr": 1000L]);
+
+    auto j = parseJSON(renderCommit(rec));
+    assert(j["schema_version"].integer == 3);
+    assert(j["commit"].str == "head1");
+    assert(j["push"]["commits"].integer == 3);
+    assert(j["metrics"]["compile_hello_debug_instr"].integer == 1000);
+    assert(j["time_trace"]["hello"]["total_us"].integer == 0);
+}