diff --git a/.github/scripts/perf_publish.sh b/.github/scripts/perf_publish.sh new file mode 100755 index 000000000000..f368ac5d95df --- /dev/null +++ b/.github/scripts/perf_publish.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Append one master measurement to the perf history repo. +# Needs PERF_DATA_REPO, PERF_DATA_KEY (deploy key with write access), +# SHA and COMMITTED_AT in the environment. +set -ueo pipefail + +results="$1" +clone="$RUNNER_TEMP/perf-data" +key=~/.ssh/perf_data + +mkdir -p ~/.ssh +printf '%s\n' "$PERF_DATA_KEY" > "$key" +chmod 600 "$key" +ssh-keyscan github.com >> ~/.ssh/known_hosts 2> /dev/null +export GIT_SSH_COMMAND="ssh -i $key -o IdentitiesOnly=yes" + +git clone "git@github.com:$PERF_DATA_REPO.git" "$clone" +cd "$clone" +git config user.name dmd-perf-bot +git config user.email dmd-perf-bot@users.noreply.github.com + +dest="data/${COMMITTED_AT:0:4}/${COMMITTED_AT:5:2}" +mkdir -p "$dest" +cp "$results" "$dest/$SHA.json" +git add "$dest/$SHA.json" + +if git diff --cached --quiet; then + echo "$SHA already published" + exit 0 +fi + +git commit -m "$SHA" + +# Another master run may have published while we were measuring. +for _ in 1 2 3; do + if git push; then + exit 0 + fi + git pull --rebase +done +exit 1 diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 207d5949448e..66fa06c8269a 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -9,9 +9,10 @@ on: push: branches: [master] +# Master runs queue instead of cancelling: every one of them is a data point. concurrency: group: perf-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -42,12 +43,24 @@ jobs: run: | set -uexo pipefail HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" - git fetch --no-tags origin master - BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)" - echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT" - echo "base=$BASE_SHA" >> "$GITHUB_OUTPUT" + echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT" echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT" + # On master the merge-base of the pushed commit is the commit itself, + # so there is nothing to diff against: measure it on its own instead. + if [ "${{ github.event_name }}" = pull_request ]; then + git fetch --no-tags origin master + echo "base=$(git merge-base "$HEAD_SHA" origin/master)" >> "$GITHUB_OUTPUT" + else + BEFORE="${{ github.event.before }}" + # Unset after a force-push or on a brand new branch. + git rev-parse -q --verify "$BEFORE^{commit}" > /dev/null || BEFORE="$HEAD_SHA~" + echo "base=" >> "$GITHUB_OUTPUT" + echo "before=$BEFORE" >> "$GITHUB_OUTPUT" + echo "commits=$(git rev-list --count "$BEFORE..$HEAD_SHA")" >> "$GITHUB_OUTPUT" + echo "date=$(git show -s --format=%cI "$HEAD_SHA")" >> "$GITHUB_OUTPUT" + fi + - name: Install prerequisites run: | sudo apt-get update @@ -67,6 +80,7 @@ jobs: source ~/dlang/*/activate built=generated/$OS_NAME/release/$MODEL profile="$RUNNER_TEMP/pgo/merged.data" + BASE_SHA="${{ steps.refs.outputs.base }}" mkdir -p "$RUNNER_TEMP/pgo" # Check out a ref and build a plain dmd + phobos @@ -88,14 +102,22 @@ jobs: DFLAGS="-fprofile-instr-use=$profile -wi" -j$N --force ) } - setup_ref "${{ steps.refs.outputs.base }}" base setup_ref "${{ steps.refs.outputs.head }}" head + # Train on base where there is one, so the profile cannot favour the PR's code. + if [ -n "$BASE_SHA" ]; then + setup_ref "$BASE_SHA" base + train=base + else + train=head + fi # Train PGO once, then reuse the profile for both refs. - ( cd "$RUNNER_TEMP/base/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force ) - cp "$RUNNER_TEMP/base/dmd/$built/dmd_profdata/merged.data" "$profile" - build_with_profile base + ( cd "$RUNNER_TEMP/$train/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force ) + cp "$RUNNER_TEMP/$train/dmd/$built/dmd_profdata/merged.data" "$profile" build_with_profile head + if [ -n "$BASE_SHA" ]; then + build_with_profile base + fi deactivate - name: Measure @@ -103,17 +125,32 @@ jobs: set -uexo pipefail source ~/dlang/*/activate built=generated/$OS_NAME/release/$MODEL/dmd - cd tools/perfrunner - dub run -- \ - --base-dmd "$RUNNER_TEMP/base/dmd/$built" \ - --head-dmd "$RUNNER_TEMP/head/dmd/$built" \ - --base-phobos "$RUNNER_TEMP/base/phobos" \ - --head-phobos "$RUNNER_TEMP/head/phobos" \ - --base-sha "${{ steps.refs.outputs.base }}" \ - --head-sha "${{ steps.refs.outputs.head }}" \ - --pr "${{ github.event.pull_request.number || 0 }}" \ - --host-dmd "$HOST_DMD" \ + BASE_SHA="${{ steps.refs.outputs.base }}" + + args=( + --head-dmd "$RUNNER_TEMP/head/dmd/$built" + --head-phobos "$RUNNER_TEMP/head/phobos" + --head-sha "${{ steps.refs.outputs.head }}" + --host-dmd "$HOST_DMD" --out "$GITHUB_WORKSPACE/results.json" + ) + if [ -n "$BASE_SHA" ]; then + args+=( + --base-dmd "$RUNNER_TEMP/base/dmd/$built" + --base-phobos "$RUNNER_TEMP/base/phobos" + --base-sha "$BASE_SHA" + --pr "${{ github.event.pull_request.number || 0 }}" + ) + else + args+=( + --before "${{ steps.refs.outputs.before }}" + --commits "${{ steps.refs.outputs.commits }}" + --committed-at "${{ steps.refs.outputs.date }}" + ) + fi + + cd tools/perfrunner + dub run -- "${args[@]}" - name: Show results run: cat "$GITHUB_WORKSPACE/results.json" @@ -125,3 +162,13 @@ jobs: with: name: perf-results path: results.json + + # Skipped until PERF_DATA_REPO is configured, so this is safe to merge early. + - name: Publish to the history repo + if: github.event_name == 'push' && vars.PERF_DATA_REPO != '' + env: + PERF_DATA_REPO: ${{ vars.PERF_DATA_REPO }} + PERF_DATA_KEY: ${{ secrets.PERF_DATA_KEY }} + SHA: ${{ steps.refs.outputs.head }} + COMMITTED_AT: ${{ steps.refs.outputs.date }} + run: .github/scripts/perf_publish.sh "$GITHUB_WORKSPACE/results.json" diff --git a/tools/perfrunner/source/app.d b/tools/perfrunner/source/app.d index ad75cd72f377..b97181155885 100644 --- a/tools/perfrunner/source/app.d +++ b/tools/perfrunner/source/app.d @@ -6,7 +6,7 @@ import std.path : buildPath, dirName; import std.stdio : stderr, writeln; import metrics : measure, initials; -import report : MetricResult, render, Report; +import report : CommitRecord, MetricResult, render, renderCommit, Report; import vibed : describeFlags; enum workloads = buildPath(__FILE_FULL_PATH__.dirName.dirName, "workloads"); @@ -21,9 +21,10 @@ version (unittest) {} else int main(string[] args) { string baseDmd, headDmd, basePhobos, headPhobos, baseSha, headSha, hostDmd; + string before, committedAt; string os = "ubuntu-latest"; string outPath = "results.json"; - long pr; + long pr, commits = 1; auto help = getopt(args, "base-dmd", "path to the base (merge-base) dmd binary", &baseDmd, @@ -33,6 +34,9 @@ int main(string[] args) "base-sha", "base commit sha (metadata)", &baseSha, "head-sha", "head commit sha (metadata)", &headSha, "pr", "pull request number (metadata)", &pr, + "before", "sha master pointed at before the push (metadata)", &before, + "commits", "commits contained in the push (metadata)", &commits, + "committed-at", "commit timestamp (metadata)", &committedAt, "os", "runner OS label (metadata)", &os, "host-dmd", "bootstrap dmd version (metadata)", &hostDmd, "out", "where to write results.json", &outPath, @@ -40,16 +44,23 @@ int main(string[] args) if (help.helpWanted) { - writeln("usage: perfrunner --base-dmd --head-dmd " - ~ "--base-phobos --head-phobos " + writeln("usage: perfrunner --head-dmd --head-phobos " + ~ "[--base-dmd --base-phobos ] " ~ "[--base-sha --head-sha --pr ] --out results.json"); + writeln("without a base the single commit is measured for the history repo"); return 0; } - if (baseDmd.length == 0 || headDmd.length == 0 - || basePhobos.length == 0 || headPhobos.length == 0) + if (headDmd.length == 0 || headPhobos.length == 0) { - stderr.writeln("error: --base-dmd, --head-dmd, --base-phobos and --head-phobos are required"); + stderr.writeln("error: --head-dmd and --head-phobos are required"); + return 2; + } + + immutable diff = baseDmd.length != 0; + if (diff && basePhobos.length == 0) + { + stderr.writeln("error: --base-dmd needs --base-phobos"); return 2; } @@ -57,7 +68,16 @@ int main(string[] args) mkdirRecurse(tmp); // Resolved once so both refs compile vibe.d with the same flags. - auto vibedFlags = describeFlags(vibedDir, baseDmd); + auto vibedFlags = describeFlags(vibedDir, diff ? baseDmd : headDmd); + + if (!diff) + { + auto m = measure(headDmd, workload, headPhobos, vibedRoot, vibedFlags, tmp, "head"); + write(outPath, renderCommit(CommitRecord(headSha, committedAt, before, commits, + os, hostDmd, m.metrics, m.helloTrace, m.phobosTrace))); + writeln("wrote ", outPath); + return 0; + } auto base = measure(baseDmd, workload, basePhobos, vibedRoot, vibedFlags, tmp, "base"); auto head = measure(headDmd, workload, headPhobos, vibedRoot, vibedFlags, tmp, "head"); diff --git a/tools/perfrunner/source/report.d b/tools/perfrunner/source/report.d index 1989734b44f5..cd1c24bd3828 100644 --- a/tools/perfrunner/source/report.d +++ b/tools/perfrunner/source/report.d @@ -29,6 +29,20 @@ struct Report Trace phobosBase, phobosHead; } +// One master commit, measured on its own. `before` and `commits` describe the +// push it arrived in, since a rebase-merged PR lands as several commits at once. +struct CommitRecord +{ + string sha; + string committedAt; + string before; + long commits; + string os; + string hostDmd; + long[string] metrics; + Trace hello, phobos; +} + // Serialise a report to the initial schema string render(Report rep) { @@ -61,6 +75,29 @@ string render(Report rep) return root.toPrettyString(); } +// Serialise a single-commit measurement for the history repo +string renderCommit(CommitRecord rec) +{ + JSONValue[string] metrics; + foreach (id, value; rec.metrics) + metrics[id] = JSONValue(value); + + JSONValue root = [ + "schema_version": JSONValue(3), + "commit": JSONValue(rec.sha), + "committed_at": JSONValue(rec.committedAt), + "push": JSONValue(["before": JSONValue(rec.before), "commits": JSONValue(rec.commits)]), + "runner": JSONValue(["os": JSONValue(rec.os), "host_dmd": JSONValue(rec.hostDmd)]), + "metrics": JSONValue(metrics), + "time_trace": JSONValue([ + "hello": traceJson(rec.hello), + "phobos": traceJson(rec.phobos), + ]), + ]; + + return root.toPrettyString(); +} + private JSONValue pair(long base, long head) { return JSONValue(["base": JSONValue(base), "head": JSONValue(head)]); @@ -78,6 +115,18 @@ private JSONValue traceJson(Trace b, Trace h) ]); } +private JSONValue traceJson(Trace t) +{ + JSONValue[string] phases; + foreach (id; phaseIds) + phases[id] = JSONValue(t.phase(id)); + + return JSONValue([ + "total_us": JSONValue(t.total), + "phases": JSONValue(phases), + ]); +} + unittest { auto rep = Report("base1", "merge-base", "head1", 7, "ubuntu-latest", "2.112.0", @@ -97,3 +146,16 @@ unittest import std.math : isClose; assert(isClose(m["delta_pct"].floating, 1.0)); } + +unittest +{ + auto rec = CommitRecord("head1", "2026-07-30T09:12:44Z", "before1", 3, + "ubuntu-latest", "ldc-1.42.0", ["compile_hello_debug_instr": 1000L]); + + auto j = parseJSON(renderCommit(rec)); + assert(j["schema_version"].integer == 3); + assert(j["commit"].str == "head1"); + assert(j["push"]["commits"].integer == 3); + assert(j["metrics"]["compile_hello_debug_instr"].integer == 1000); + assert(j["time_trace"]["hello"]["total_us"].integer == 0); +}