[VC-59470] Run the e2e suites against master every night

.github/workflows/e2e.yaml

@@ -17,6 +17,13 @@ on:
   # do: they read github.event.pull_request.labels, which is empty for a push.
   # Needed before tagging a release.
   workflow_dispatch: {}
+  # Run against master every night, so that a green master means the
+  # end-to-end tests actually passed. Nothing else runs them on master: the
+  # label gates read github.event.pull_request.labels, which is empty outside
+  # a pull request. 02:00 UTC leaves midnight to govulncheck, and puts the
+  # result in before the working day.
+  schedule:
+    - cron: '0 2 * * *'
 jobs:
   ark-test-e2e:
     # TEMPORARY: require an explicit label to test disco-agent until the test environment fixes a recurring issue
@@ -26,11 +33,15 @@ jobs:
     # Runs when the label is added, and thereafter on every push while it is
     # still on the pull request. `github.event.label` names only the label that
     # was just added, so adding one e2e label does not start the other suites.
+    # Outside a pull request there are no labels to read, so the schedule and
+    # the Run workflow button always run everything. The repository check keeps
+    # the nightly from running in forks, as govulncheck.yaml does.
     if: >-
-      github.event_name == 'workflow_dispatch'
+      github.repository == 'jetstack/jetstack-secure'
+      && (github.event_name != 'pull_request'
       || github.event.label.name == 'test-ark'
       || (github.event.action != 'labeled'
-      && contains(github.event.pull_request.labels.*.name, 'test-ark'))
+      && contains(github.event.pull_request.labels.*.name, 'test-ark')))
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -70,10 +81,11 @@ jobs:
     # TEMPORARY: require an explicit label to test NGTS until we have a stable test environment
     # See `ark-test-e2e`.
     if: >-
-      github.event_name == 'workflow_dispatch'
+      github.repository == 'jetstack/jetstack-secure'
+      && (github.event_name != 'pull_request'
       || github.event.label.name == 'test-ngts'
       || (github.event.action != 'labeled'
-      && contains(github.event.pull_request.labels.*.name, 'test-ngts'))
+      && contains(github.event.pull_request.labels.*.name, 'test-ngts')))
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -110,10 +122,11 @@ jobs:
   test-e2e:
     # See `ark-test-e2e`.
     if: >-
-      github.event_name == 'workflow_dispatch'
+      github.repository == 'jetstack/jetstack-secure'
+      && (github.event_name != 'pull_request'
       || github.event.label.name == 'test-e2e'
       || (github.event.action != 'labeled'
-      && contains(github.event.pull_request.labels.*.name, 'test-e2e'))
+      && contains(github.event.pull_request.labels.*.name, 'test-e2e')))
     runs-on: ubuntu-latest
     # A healthy run takes about 15 minutes. The backstop matters because the job
     # holds a GKE cluster for as long as it runs, and the default is 6 hours.
@@ -193,3 +206,59 @@ jobs:
             --project=machineidentitysecurity-jsci-e \
             --zone=europe-west1-b \
             --quiet
+
+  notify:
+    # Only the nightly needs to announce itself. A suite that fails on a
+    # labelled pull request is already in front of whoever added the label,
+    # and a Slack message for it would be noise.
+    #
+    # One message for the whole run, rather than one per suite, so a bad night
+    # is a single notification.
+    #
+    # `needs.<job>.result` is one of success, failure, cancelled or skipped. A
+    # cancelled suite has not passed, and a nightly that never finished is as
+    # uninformative as one that failed, so treat it the same. `!cancelled()`
+    # rather than `always()` so that cancelling the whole run does not make it
+    # send a message about itself.
+    if: >-
+      !cancelled()
+      && github.event_name == 'schedule'
+      && (contains(needs.*.result, 'failure')
+      || contains(needs.*.result, 'cancelled'))
+    needs: [ark-test-e2e, ngts-test-e2e, test-e2e]
+    runs-on: ubuntu-latest
+    # The default job timeout is 6 hours. Nothing here should take a minute.
+    timeout-minutes: 5
+    steps:
+      - name: Report the failure to Slack
+        env:
+          # Not yet set on this repository. Until it is, the run says so in the
+          # job summary rather than failing, so a missing secret cannot turn a
+          # passing nightly red.
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          ARK: ${{ needs.ark-test-e2e.result }}
+          NGTS: ${{ needs.ngts-test-e2e.result }}
+          GKE: ${{ needs.test-e2e.result }}
+        run: |
+          RESULTS="ark: ${ARK}, ngts: ${NGTS}, gke: ${GKE}"
+
+          # The summary is Markdown and Slack is mrkdwn; the two link syntaxes
+          # are not interchangeable. While SLACK_WEBHOOK is unset the summary is
+          # the only place the result is written, so its link has to work.
+          echo ":x: Nightly e2e did not pass on master — ${RESULTS} ([View run](${RUN_URL}))" >> "$GITHUB_STEP_SUMMARY"
+
+          if [ -z "${SLACK_WEBHOOK}" ]; then
+            echo "::warning::SLACK_WEBHOOK is not set on this repository, so the nightly e2e failure was not sent to Slack"
+            exit 0
+          fi
+
+          TEXT=":x: Nightly e2e did not pass on master — ${RESULTS} (<${RUN_URL}|View run>)"
+
+          # --max-time because curl has no overall limit of its own, and a
+          # connection that stalls after being accepted would otherwise hold the
+          # job open. Matches the polling loop in hack/e2e/test.sh.
+          curl --fail --silent --show-error --max-time 30 -X POST \
+            -H 'Content-type: application/json' \
+            --data "$(jq --null-input --arg text "${TEXT}" '{text: $text}')" \
+            "${SLACK_WEBHOOK}"

CONTRIBUTING.md

@@ -175,7 +175,8 @@ E2E tests run automatically in CI when you add specific labels to your PR:
 - Add `keep-e2e-cluster` **before** `test-e2e`. Applying `test-e2e` starts the run straight away, and the run only sees the labels that were set at that moment, so adding `keep-e2e-cluster` afterwards will not save the cluster.
 
 The suites live in [.github/workflows/e2e.yaml](./.github/workflows/e2e.yaml). You can also run them against any branch
-without a label, using the **Run workflow** button on the `e2e` workflow.
+without a label, using the **Run workflow** button on the `e2e` workflow. They also run against `master` every night at
+02:00 UTC. If a suite fails on your pull request, check the most recent nightly first: `master` may already be broken.
 
 The E2E test script is located at [hack/e2e/test.sh](./hack/e2e/test.sh).