Test the runner images in a dedicated runner-images PR check

.github/codeql/codeql-config-actions.yml

@@ -0,0 +1,5 @@
+paths-ignore:
+  - lib
+  - tests
+queries:
+  - uses: security-and-quality

.github/workflows/codeql.yml

@@ -19,69 +19,16 @@ env:
   CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks
 
 jobs:
-  # Identify the CodeQL tool versions to use in the analysis job.
-  check-codeql-versions:
+  analyze:
     if: github.triggering_actor != 'dependabot[bot]'
     runs-on: ubuntu-26.04
-    outputs:
-      versions: ${{ steps.compare.outputs.versions }}
 
-    permissions:
-      contents: read
-      # We currently need `security-events: read` to access feature flags.
-      security-events: read
-
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Set up default CodeQL bundle
-        id: setup-default
-        uses: ./setup-codeql
-      - name: Set up linked CodeQL bundle
-        id: setup-linked
-        uses: ./setup-codeql
-        with:
-          tools: linked
-      - name: Compare default and linked CodeQL bundle versions
-        id: compare
-        env:
-          CODEQL_DEFAULT: ${{ steps.setup-default.outputs.codeql-path }}
-          CODEQL_LINKED: ${{ steps.setup-linked.outputs.codeql-path }}
-        run: |
-          CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)"
-          CODEQL_VERSION_LINKED="$("$CODEQL_LINKED" version --format terse)"
-          echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT"
-          echo "Linked CodeQL bundle version is $CODEQL_VERSION_LINKED"
-
-          # If we're running on a pull request, run with both bundles, even if `tools: linked` would
-          # be the same as `tools: null`. This allows us to make the job for each of the bundles a
-          # required status check.
-          #
-          # If we're running on push or schedule, then we can skip running with `tools: linked` when it would be
-          # the same as running with `tools: null`.
-          if [[ "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "merge_group" && "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LINKED" ]]; then
-            VERSIONS_JSON='[null]'
-          else
-            VERSIONS_JSON='[null, "linked"]'
-          fi
-
-          # Output a JSON-encoded list with the distinct versions to test against.
-          echo "Suggested matrix config for analysis job: $VERSIONS_JSON"
-          echo "versions=${VERSIONS_JSON}" >> $GITHUB_OUTPUT
-
-  analyze-javascript:
-    if: github.triggering_actor != 'dependabot[bot]'
-    needs: [check-codeql-versions]
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - ubuntu-22.04
-          - ubuntu-24.04
-          - windows-2022
-          - windows-2025
-          - macos-15-xlarge
-        tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }}
-    runs-on: ${{ matrix.os }}
+        language:
+          - actions
+          - javascript
 
     permissions:
       contents: read
@@ -94,49 +41,15 @@ jobs:
         uses: ./init
         id: init
         with:
-          languages: javascript
-          config-file: ./.github/codeql/codeql-config-javascript.yml
-          tools: ${{ matrix.tools }}
+          languages: ${{ matrix.language }}
+          build-mode: none
+          config-file: ./.github/codeql/codeql-config-${{ matrix.language }}.yml
       # confirm steps.init.outputs.codeql-path points to the codeql binary
       - name: Print CodeQL Version
         run: >
           "$CODEQL" version --format=json
         env:
           CODEQL: ${{steps.init.outputs.codeql-path}}
-      - name: Perform CodeQL Analysis
-        uses: ./analyze
-        with:
-          category: "/language:javascript"
-          upload: ${{ (matrix.os == 'ubuntu-24.04' && !matrix.tools && github.event_name != 'merge_group' && 'always' ) || 'never' }}
-
-  analyze-other:
-    if: github.triggering_actor != 'dependabot[bot]'
-    runs-on: ubuntu-26.04
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - language: actions
-
-    permissions:
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Initialize CodeQL
-        uses: ./init
-        with:
-          languages: ${{ matrix.language }}
-          build-mode: none
-          config: >
-            paths-ignore:
-              - lib
-              - tests
-            queries:
-              - uses: security-and-quality
       - name: Perform CodeQL Analysis
         uses: ./analyze
         with:

.github/workflows/deflake.yml

@@ -61,6 +61,7 @@ on:
       - "PR Check - Remote config file"
       - "PR Check - Resolve environment"
       - "PR Check - Ruby analysis"
+      - "PR Check - Runner images"
       - "PR Check - Rust analysis"
       - "PR Check - Split workflow"
       - "PR Check - Start proxy"

pr-checks/checks/runner-images.yml

@@ -0,0 +1,66 @@
+name: "Runner images"
+description: "An end-to-end integration test checking that CodeQL can analyze interpreted and compiled code on the GitHub-hosted runner images"
+# This check runs on each image listed in
+# https://github.com/actions/runner-images#available-images, except for:
+# - `ubuntu-slim`, which is intended for lightweight jobs rather than builds like this one.
+# - `windows-11-arm`, since CodeQL doesn't support Windows on Arm64.
+# - `xcode-27`, a preview image that the `unsupported-macos` check runs on.
+operatingSystems:
+  # The Ubuntu 22.04 images have brownouts from 23 March 2027 and are retired on 17 April 2027, so
+  # remove them before the brownouts start.
+  # See https://github.com/actions/runner-images/issues/14254.
+  - os: ubuntu
+    runner-image: ubuntu-22.04
+  - os: ubuntu
+    runner-image: ubuntu-22.04-arm
+  - os: ubuntu
+    runner-image: ubuntu-24.04
+  - os: ubuntu
+    runner-image: ubuntu-24.04-arm
+  - os: ubuntu
+    runner-image: ubuntu-26.04
+  - os: ubuntu
+    runner-image: ubuntu-26.04-arm
+  - os: windows
+    runner-image: windows-2022
+  - os: windows
+    runner-image: windows-2025
+  # Use larger runners for macOS to keep these jobs off the critical path.
+  - os: macos
+    runner-image: macos-15-large
+  - os: macos
+    runner-image: macos-15-xlarge
+  - os: macos
+    runner-image: macos-26-large
+  - os: macos
+    runner-image: macos-26-xlarge
+# Other PR checks cover the supported CodeQL versions, so this check only tests the linked version.
+# Linux Arm64 bundles are only available for CodeQL 2.27.0 and later, so older stable versions
+# can't run on the Arm64 images.
+versions:
+  - linked
+installGo: true
+installDotNet: true
+# The languages to build and analyze. CodeQL only supports Swift on macOS.
+env:
+  LANGUAGES: "cpp,csharp,go,java,javascript,python,ruby${{ startsWith(matrix.os, 'macos') && ',swift' || '' }}"
+steps:
+  - uses: ./../action/init
+    with:
+      languages: ${{ env.LANGUAGES }}
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+  - name: Build code
+    run: ./build.sh
+  - uses: ./../action/analyze
+    with:
+      upload-database: false
+  - name: Assert databases exist
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      for lang in ${LANGUAGES//,/ }; do
+        if [[ ! -d "$lang" ]]; then
+          echo "Did not find a database for $lang"
+          exit 1
+        fi
+        echo "Found database for $lang"
+      done