From eca2a5bc880cbb71184b326e6b46c6089b2ed7d3 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 18 Sep 2026 14:37:47 +0000 Subject: [PATCH] [VC-59470] Run the e2e suites against master every night Nothing runs the end-to-end suites on master. They are reachable only by labelling a pull request or by pressing Run workflow, so a green master says nothing about whether they pass, and the last three manual runs show why that matters. Add a 02:00 UTC schedule to e2e.yaml. Midnight is already taken by govulncheck, and 02:00 puts the result in before the working day. The job gates keyed on `github.event_name == 'workflow_dispatch'` become `github.event_name != 'pull_request'`. Outside a pull request there are no labels to read, so every suite should run; this covers the schedule and the Run workflow button together, and does not change what happens on a pull request. Each gate also gains the `github.repository` check that govulncheck.yaml uses, so the nightly does not run in forks. Add a notify job for the nightly only. A suite that fails on a labelled pull request is already in front of whoever added the label, so only the schedule posts to Slack, and it posts once for the whole run rather than once per suite. A cancelled suite counts as not passing, since a nightly that never finished is as uninformative as one that failed. SLACK_WEBHOOK is not set on this repository yet. Until it is, the job records the failure in the run summary and warns, so a missing secret cannot turn a passing nightly red. The summary is Markdown and Slack is mrkdwn, so the two carry different link syntax for the same run URL. Expect the first nights to be red. test-e2e has failed on master on every manual run since 2026-09-16, caused by CIS dropping async Kafka messages rather than by anything here. ark-test-e2e and ngts-test-e2e have passed on all three, which is the first real evidence that the "conflicting tagging values" flake behind the TEMPORARY comment on ark-test-e2e has gone. Co-Authored-By: Claude Signed-off-by: Richard Wall --- .github/workflows/e2e.yaml | 81 +++++++++++++++++++++++++++++++++++--- CONTRIBUTING.md | 3 +- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 55d778c5..6bbbd23b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -17,6 +17,13 @@ on: # do: they read github.event.pull_request.labels, which is empty for a push. # Needed before tagging a release. workflow_dispatch: {} + # Run against master every night, so that a green master means the + # end-to-end tests actually passed. Nothing else runs them on master: the + # label gates read github.event.pull_request.labels, which is empty outside + # a pull request. 02:00 UTC leaves midnight to govulncheck, and puts the + # result in before the working day. + schedule: + - cron: '0 2 * * *' jobs: ark-test-e2e: # TEMPORARY: require an explicit label to test disco-agent until the test environment fixes a recurring issue @@ -26,11 +33,15 @@ jobs: # Runs when the label is added, and thereafter on every push while it is # still on the pull request. `github.event.label` names only the label that # was just added, so adding one e2e label does not start the other suites. + # Outside a pull request there are no labels to read, so the schedule and + # the Run workflow button always run everything. The repository check keeps + # the nightly from running in forks, as govulncheck.yaml does. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-ark' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-ark')) + && contains(github.event.pull_request.labels.*.name, 'test-ark'))) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -70,10 +81,11 @@ jobs: # TEMPORARY: require an explicit label to test NGTS until we have a stable test environment # See `ark-test-e2e`. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-ngts' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-ngts')) + && contains(github.event.pull_request.labels.*.name, 'test-ngts'))) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -110,10 +122,11 @@ jobs: test-e2e: # See `ark-test-e2e`. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-e2e' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-e2e')) + && contains(github.event.pull_request.labels.*.name, 'test-e2e'))) runs-on: ubuntu-latest # A healthy run takes about 15 minutes. The backstop matters because the job # holds a GKE cluster for as long as it runs, and the default is 6 hours. @@ -193,3 +206,59 @@ jobs: --project=machineidentitysecurity-jsci-e \ --zone=europe-west1-b \ --quiet + + notify: + # Only the nightly needs to announce itself. A suite that fails on a + # labelled pull request is already in front of whoever added the label, + # and a Slack message for it would be noise. + # + # One message for the whole run, rather than one per suite, so a bad night + # is a single notification. + # + # `needs..result` is one of success, failure, cancelled or skipped. A + # cancelled suite has not passed, and a nightly that never finished is as + # uninformative as one that failed, so treat it the same. `!cancelled()` + # rather than `always()` so that cancelling the whole run does not make it + # send a message about itself. + if: >- + !cancelled() + && github.event_name == 'schedule' + && (contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled')) + needs: [ark-test-e2e, ngts-test-e2e, test-e2e] + runs-on: ubuntu-latest + # The default job timeout is 6 hours. Nothing here should take a minute. + timeout-minutes: 5 + steps: + - name: Report the failure to Slack + env: + # Not yet set on this repository. Until it is, the run says so in the + # job summary rather than failing, so a missing secret cannot turn a + # passing nightly red. + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ARK: ${{ needs.ark-test-e2e.result }} + NGTS: ${{ needs.ngts-test-e2e.result }} + GKE: ${{ needs.test-e2e.result }} + run: | + RESULTS="ark: ${ARK}, ngts: ${NGTS}, gke: ${GKE}" + + # The summary is Markdown and Slack is mrkdwn; the two link syntaxes + # are not interchangeable. While SLACK_WEBHOOK is unset the summary is + # the only place the result is written, so its link has to work. + echo ":x: Nightly e2e did not pass on master — ${RESULTS} ([View run](${RUN_URL}))" >> "$GITHUB_STEP_SUMMARY" + + if [ -z "${SLACK_WEBHOOK}" ]; then + echo "::warning::SLACK_WEBHOOK is not set on this repository, so the nightly e2e failure was not sent to Slack" + exit 0 + fi + + TEXT=":x: Nightly e2e did not pass on master — ${RESULTS} (<${RUN_URL}|View run>)" + + # --max-time because curl has no overall limit of its own, and a + # connection that stalls after being accepted would otherwise hold the + # job open. Matches the polling loop in hack/e2e/test.sh. + curl --fail --silent --show-error --max-time 30 -X POST \ + -H 'Content-type: application/json' \ + --data "$(jq --null-input --arg text "${TEXT}" '{text: $text}')" \ + "${SLACK_WEBHOOK}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02f1f672..5aa76ee8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -175,7 +175,8 @@ E2E tests run automatically in CI when you add specific labels to your PR: - Add `keep-e2e-cluster` **before** `test-e2e`. Applying `test-e2e` starts the run straight away, and the run only sees the labels that were set at that moment, so adding `keep-e2e-cluster` afterwards will not save the cluster. The suites live in [.github/workflows/e2e.yaml](./.github/workflows/e2e.yaml). You can also run them against any branch -without a label, using the **Run workflow** button on the `e2e` workflow. +without a label, using the **Run workflow** button on the `e2e` workflow. They also run against `master` every night at +02:00 UTC. If a suite fails on your pull request, check the most recent nightly first: `master` may already be broken. The E2E test script is located at [hack/e2e/test.sh](./hack/e2e/test.sh).