Next main push: the software images, the toolkit and the browser

.github/actions/prepare/action.yml

@@ -5,7 +5,10 @@ inputs:
     description: Also let the runner start VMs (KVM, and virt-fw-vars for the variable stores).
     default: 'false'
   sources:
-    description: The upstream sources, cached by sources.lock. Either fetches what the cache lacks; 'fetch' also saves the set, on main.
+    description: The root's upstream sources, cached by sources.lock. Either fetches what the cache lacks; 'fetch' also saves the set, on main.
+    default: ''
+  image-sources:
+    description: The software images' sources too, a cache of their own keyed by build/images/sources.lock; 'fetch' or 'restore' as for sources.
     default: ''
 runs:
   using: composite
@@ -35,6 +38,9 @@ runs:
         sudo udevadm trigger --name-match=kvm
         ls -la /dev/kvm
 
+    # The root's set, keyed by its lock alone. images/, the software images' sources, has a cache of
+    # its own and is set aside while this one is saved: a browser bump then copies none of the root's
+    # set, and a root bump none of theirs. The path is the one it always had, so earlier sets restore.
     - if: inputs.sources != ''
       id: sources
       uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
@@ -44,22 +50,53 @@ runs:
         # A changed lock restores the previous set; every file is still hashed against the lock.
         restore-keys: sources-
 
-    # A cache evicted since the build job is fetched again, not taken as gone.
-    - if: inputs.sources != '' && steps.sources.outputs.cache-hit != 'true'
+    # With the signatures, which verify-signatures keeps beside the root's: when an image's pin alone
+    # moves, only this cache is saved again.
+    - if: inputs.image-sources != ''
+      id: image_sources
+      uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+      with:
+        path: |
+          /mnt/kryptik/sources/images
+          /mnt/kryptik/sources/.signatures
+        key: image-sources-v1-${{ hashFiles('build/images/sources.lock') }}
+        restore-keys: image-sources-v1-
+
+    # A cache evicted since the build job is fetched again, not taken as gone. With the images' sources
+    # this runs on two hits as well: the root's set restored may be one saved without the links that
+    # name each image source in sources/ (main's, or an older set's), and only this pass makes them.
+    - if: (inputs.sources != '' && steps.sources.outputs.cache-hit != 'true') || inputs.image-sources != ''
       shell: bash
       env:
-        MODE: ${{ inputs.sources }}
+        IMAGES: ${{ inputs.image-sources }}
+        SAVES: ${{ (inputs.sources == 'fetch' && steps.sources.outputs.cache-hit != 'true') || (inputs.image-sources == 'fetch' && steps.image_sources.outputs.cache-hit != 'true') }}
       run: |
-        make sources
+        if [[ -n "$IMAGES" ]]; then make sources; else ./tools/fetch-sources.sh --root; fi
         ./tools/prune-sources.sh
-        # Fetches the signatures into the saved set for CI's gate; this pass is not the verdict.
-        [[ "$MODE" != fetch ]] || ./tools/verify-signatures.sh > /dev/null 2>&1 || true
+        # Fetches the signatures into the set saved below for CI's gate; this pass is not the verdict.
+        [[ "$SAVES" != true ]] || ./tools/verify-signatures.sh > /dev/null 2>&1 || true
 
     # Saved now rather than at the end, so a job that fails later does not refetch the new files.
     # From main alone, which every branch's run restores: a branch whose lock adds one file would
     # save the whole set again in its own scope, and such copies pushed main's caches out.
+    - if: inputs.sources == 'fetch' && steps.sources.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+      shell: bash
+      run: if [[ -d /mnt/kryptik/sources/images ]]; then mv /mnt/kryptik/sources/images /mnt/kryptik/images-aside; fi
+
     - if: inputs.sources == 'fetch' && steps.sources.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
       uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
       with:
         path: /mnt/kryptik/sources
         key: sources-v3-${{ hashFiles('sources.lock') }}
+
+    - if: always() && inputs.sources == 'fetch' && steps.sources.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+      shell: bash
+      run: if [[ -d /mnt/kryptik/images-aside ]]; then mv /mnt/kryptik/images-aside /mnt/kryptik/sources/images; fi
+
+    - if: inputs.image-sources == 'fetch' && steps.image_sources.outputs.cache-hit != 'true'
+      uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+      with:
+        path: |
+          /mnt/kryptik/sources/images
+          /mnt/kryptik/sources/.signatures
+        key: image-sources-v1-${{ hashFiles('build/images/sources.lock') }}

.github/workflows/ci.yml

@@ -125,7 +125,9 @@ jobs:
           sudo mkdir -p /mnt/kryptik && sudo chown "$(id -u):$(id -g)" /mnt/kryptik
           ./tools/fetch-sources.sh --list
 
-      # Restores the last set; `make sources` checks it against the lock and fetches what is missing.
+      # Restores the last sets; `make sources` checks them against the locks and fetches what is missing.
+      # The software images' sources, under images/, are a cache of their own (the prepare action's),
+      # set aside while the root's is saved.
       - name: Upstream sources (cached by sources.lock)
         id: sources
         uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
@@ -135,23 +137,56 @@ jobs:
           restore-keys: |
             sources-
 
-      # Fetches the signatures into the cached set for the gate below; this pass is not the verdict.
+      # With the signatures, which verify-signatures keeps beside the root's: when an image's pin alone
+      # moves, only this cache is saved again.
+      - name: The software images' sources (cached by build/images/sources.lock)
+        id: image_sources
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: |
+            /mnt/kryptik/sources/images
+            /mnt/kryptik/sources/.signatures
+          key: image-sources-v1-${{ hashFiles('build/images/sources.lock') }}
+          restore-keys: |
+            image-sources-v1-
+
+      # Runs on two hits as well: the root's set restored may be one saved without the links that name
+      # each image source in sources/ (an older set's), and only this pass makes them. On a miss it
+      # fetches the signatures into the cached set for the gate below; that pass is not the verdict.
       - name: Fetch and verify the sources
-        if: steps.sources.outputs.cache-hit != 'true'
+        env:
+          MISSED: ${{ steps.sources.outputs.cache-hit != 'true' || steps.image_sources.outputs.cache-hit != 'true' }}
         run: |
           make sources
           ./tools/prune-sources.sh
-          ./tools/verify-signatures.sh > /dev/null 2>&1 || true
+          [[ "$MISSED" != true ]] || ./tools/verify-signatures.sh > /dev/null 2>&1 || true
 
       # From main alone, as the Distro jobs save it: a pull request's cache is scoped to its merge
       # ref and would duplicate the set.
+      - name: Set the software images' sources aside
+        if: ${{ github.ref == 'refs/heads/main' && steps.sources.outputs.cache-hit != 'true' }}
+        run: if [[ -d /mnt/kryptik/sources/images ]]; then mv /mnt/kryptik/sources/images /mnt/kryptik/images-aside; fi
+
       - name: Save the sources
         if: ${{ github.ref == 'refs/heads/main' && steps.sources.outputs.cache-hit != 'true' }}
         uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
         with:
           path: /mnt/kryptik/sources
           key: ${{ steps.sources.outputs.cache-primary-key }}
 
+      - name: The software images' sources back in place
+        if: always()
+        run: if [[ -d /mnt/kryptik/images-aside ]]; then mv /mnt/kryptik/images-aside /mnt/kryptik/sources/images; fi
+
+      - name: Save the software images' sources
+        if: ${{ github.event_name != 'pull_request' && steps.image_sources.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: |
+            /mnt/kryptik/sources/images
+            /mnt/kryptik/sources/.signatures
+          key: ${{ steps.image_sources.outputs.cache-primary-key }}
+
       # Informational, as mirrors go down; a one-byte ranged GET, since some GNU mirrors refuse HEAD.
       - name: Every source URL is reachable
         continue-on-error: true
@@ -206,17 +241,22 @@ jobs:
       - name: Signatures, strict
         run: ./tools/verify-signatures.sh --strict
 
+      # The root's sources in sources.lock, the software images' in build/images/sources.lock.
       - name: sources.lock covers the manifest
         run: |
           fail=0
-          while read -r name ver url _; do
-            file=$(basename "$url")
-            if ! grep -q "  $file\$" sources.lock; then
-              echo "  MISSING FROM LOCK: $file"
-              fail=1
-            fi
-          done < <(./tools/fetch-sources.sh --list)
-          [ $fail -eq 0 ] && echo "sources.lock covers every manifest entry"
+          check() {
+            while read -r name ver url _; do
+              file=$(basename "$url")
+              if ! grep -q "  $file\$" "$1"; then
+                echo "  MISSING FROM $1: $file"
+                fail=1
+              fi
+            done
+          }
+          check sources.lock < <(./tools/fetch-sources.sh --list-root)
+          check build/images/sources.lock < <(./tools/fetch-sources.sh --list | grep -vxF -f <(./tools/fetch-sources.sh --list-root))
+          [ $fail -eq 0 ] && echo "sources.lock and build/images/sources.lock cover every manifest entry"
           exit $fail
 
   compartments: