Repository navigation
143 lines (126 loc) · 6.1 KB
/
Copy pathbenchmarks.yml
File metadata and controls
143 lines (126 loc) · 6.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
name: Python SDK benchmarks
on: [pull_request]
permissions: {}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
benchmark:
runs-on: ubuntu-24.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
shard: [flags-single, flags-scaling, flags-memory, capture]
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.12.12'
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
version: '0.13.4'
enable-cache: true
- name: Prepare pinned baseline
env:
TARGET_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -euo pipefail
export BASE_WORKTREE="$RUNNER_TEMP/python-benchmark-base"
export RESULTS="$RUNNER_TEMP/python-benchmark-results"
printf 'BASE_WORKTREE=%s\nRESULTS=%s\n' "$BASE_WORKTREE" "$RESULTS" >> "$GITHUB_ENV"
BASE_SHA=$(git merge-base "$TARGET_SHA" "$HEAD_SHA")
git worktree add --detach "$BASE_WORKTREE" "$BASE_SHA"
mkdir -p "$RESULTS"
python -c 'import json, os, pathlib, sys; pathlib.Path(os.environ["RESULTS"], "revisions.json").write_text(json.dumps({"target":os.environ["TARGET_SHA"], "head":os.environ["HEAD_SHA"], "base":sys.argv[1]}, indent=2))' "$BASE_SHA"
- name: Install isolated head and baseline wheels
run: |
python benchmarks/prepare.py .
python benchmarks/prepare.py "$BASE_WORKTREE"
- name: Test measurement controls and fixtures
run: .venv-benchmark/bin/python -B benchmarks/test_benchmarks.py
- name: Measure paired base and head
id: measure
timeout-minutes: 8
run: |
.venv-benchmark/bin/python -B benchmarks/run.py \
--base "$BASE_WORKTREE" \
--head "$GITHUB_WORKSPACE" \
--output "$RESULTS" \
--repetitions 6 \
--shard "${{ matrix.shard }}"
- name: Upload raw shard results
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: benchmark-shard-${{ matrix.shard }}
path: |
${{ runner.temp }}/python-benchmark-results/base.json
${{ runner.temp }}/python-benchmark-results/head.json
${{ runner.temp }}/python-benchmark-results/revisions.json
if-no-files-found: warn
retention-days: 14
collect:
needs: benchmark
if: ${{ !cancelled() }}
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- name: Download shard results
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: benchmark-shard-*
path: ${{ runner.temp }}/python-benchmark-shards
merge-multiple: false
- name: Collect and render benchmark comparison
uses: PostHog/.github/.github/actions/benchmark-report@3322f26f21f088ad5e5960d65a916971e309ac0b # SDK benchmark reporting (PostHog/.github PR #84)
with:
mode: collect
source-status: ${{ needs.benchmark.result }}
shard-manifest: benchmarks/shards.json
shard-directory: ${{ runner.temp }}/python-benchmark-shards
output-directory: ${{ runner.temp }}/python-benchmark-report
- name: Upload rendered comparison
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: python-benchmark-report
path: |
${{ runner.temp }}/python-benchmark-report/report.md
${{ runner.temp }}/python-benchmark-report/comparison.json
if-no-files-found: ignore
retention-days: 14
- name: Upload raw benchmark results
if: always()
id: results
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: python-benchmark-results
path: |
${{ runner.temp }}/python-benchmark-report/bundle.json
${{ runner.temp }}/python-benchmark-report/revisions.json
if-no-files-found: warn
retention-days: 14
- name: Link results
if: always()
env:
ARTIFACT_URL: ${{ steps.results.outputs.artifact-url }}
run: |
printf '## Python SDK benchmark data\n\n' >> "$GITHUB_STEP_SUMMARY"
if [[ -n "$ARTIFACT_URL" ]]; then
printf '[Download paired measurements and execution metadata](%s)\n\n' "$ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY"
fi
printf 'Measurements are informational. Check workload statuses before comparing values.\n' >> "$GITHUB_STEP_SUMMARY"