diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 55d778c5..6bbbd23b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -17,6 +17,13 @@ on: # do: they read github.event.pull_request.labels, which is empty for a push. # Needed before tagging a release. workflow_dispatch: {} + # Run against master every night, so that a green master means the + # end-to-end tests actually passed. Nothing else runs them on master: the + # label gates read github.event.pull_request.labels, which is empty outside + # a pull request. 02:00 UTC leaves midnight to govulncheck, and puts the + # result in before the working day. + schedule: + - cron: '0 2 * * *' jobs: ark-test-e2e: # TEMPORARY: require an explicit label to test disco-agent until the test environment fixes a recurring issue @@ -26,11 +33,15 @@ jobs: # Runs when the label is added, and thereafter on every push while it is # still on the pull request. `github.event.label` names only the label that # was just added, so adding one e2e label does not start the other suites. + # Outside a pull request there are no labels to read, so the schedule and + # the Run workflow button always run everything. The repository check keeps + # the nightly from running in forks, as govulncheck.yaml does. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-ark' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-ark')) + && contains(github.event.pull_request.labels.*.name, 'test-ark'))) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -70,10 +81,11 @@ jobs: # TEMPORARY: require an explicit label to test NGTS until we have a stable test environment # See `ark-test-e2e`. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-ngts' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-ngts')) + && contains(github.event.pull_request.labels.*.name, 'test-ngts'))) runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -110,10 +122,11 @@ jobs: test-e2e: # See `ark-test-e2e`. if: >- - github.event_name == 'workflow_dispatch' + github.repository == 'jetstack/jetstack-secure' + && (github.event_name != 'pull_request' || github.event.label.name == 'test-e2e' || (github.event.action != 'labeled' - && contains(github.event.pull_request.labels.*.name, 'test-e2e')) + && contains(github.event.pull_request.labels.*.name, 'test-e2e'))) runs-on: ubuntu-latest # A healthy run takes about 15 minutes. The backstop matters because the job # holds a GKE cluster for as long as it runs, and the default is 6 hours. @@ -193,3 +206,59 @@ jobs: --project=machineidentitysecurity-jsci-e \ --zone=europe-west1-b \ --quiet + + notify: + # Only the nightly needs to announce itself. A suite that fails on a + # labelled pull request is already in front of whoever added the label, + # and a Slack message for it would be noise. + # + # One message for the whole run, rather than one per suite, so a bad night + # is a single notification. + # + # `needs..result` is one of success, failure, cancelled or skipped. A + # cancelled suite has not passed, and a nightly that never finished is as + # uninformative as one that failed, so treat it the same. `!cancelled()` + # rather than `always()` so that cancelling the whole run does not make it + # send a message about itself. + if: >- + !cancelled() + && github.event_name == 'schedule' + && (contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled')) + needs: [ark-test-e2e, ngts-test-e2e, test-e2e] + runs-on: ubuntu-latest + # The default job timeout is 6 hours. Nothing here should take a minute. + timeout-minutes: 5 + steps: + - name: Report the failure to Slack + env: + # Not yet set on this repository. Until it is, the run says so in the + # job summary rather than failing, so a missing secret cannot turn a + # passing nightly red. + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ARK: ${{ needs.ark-test-e2e.result }} + NGTS: ${{ needs.ngts-test-e2e.result }} + GKE: ${{ needs.test-e2e.result }} + run: | + RESULTS="ark: ${ARK}, ngts: ${NGTS}, gke: ${GKE}" + + # The summary is Markdown and Slack is mrkdwn; the two link syntaxes + # are not interchangeable. While SLACK_WEBHOOK is unset the summary is + # the only place the result is written, so its link has to work. + echo ":x: Nightly e2e did not pass on master — ${RESULTS} ([View run](${RUN_URL}))" >> "$GITHUB_STEP_SUMMARY" + + if [ -z "${SLACK_WEBHOOK}" ]; then + echo "::warning::SLACK_WEBHOOK is not set on this repository, so the nightly e2e failure was not sent to Slack" + exit 0 + fi + + TEXT=":x: Nightly e2e did not pass on master — ${RESULTS} (<${RUN_URL}|View run>)" + + # --max-time because curl has no overall limit of its own, and a + # connection that stalls after being accepted would otherwise hold the + # job open. Matches the polling loop in hack/e2e/test.sh. + curl --fail --silent --show-error --max-time 30 -X POST \ + -H 'Content-type: application/json' \ + --data "$(jq --null-input --arg text "${TEXT}" '{text: $text}')" \ + "${SLACK_WEBHOOK}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02f1f672..5aa76ee8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -175,7 +175,8 @@ E2E tests run automatically in CI when you add specific labels to your PR: - Add `keep-e2e-cluster` **before** `test-e2e`. Applying `test-e2e` starts the run straight away, and the run only sees the labels that were set at that moment, so adding `keep-e2e-cluster` afterwards will not save the cluster. The suites live in [.github/workflows/e2e.yaml](./.github/workflows/e2e.yaml). You can also run them against any branch -without a label, using the **Run workflow** button on the `e2e` workflow. +without a label, using the **Run workflow** button on the `e2e` workflow. They also run against `master` every night at +02:00 UTC. If a suite fails on your pull request, check the most recent nightly first: `master` may already be broken. The E2E test script is located at [hack/e2e/test.sh](./hack/e2e/test.sh).