CI: real-tool smoke for tflint, kics, cppcheck and devskim

.github/workflows/ci.yml

@@ -515,10 +515,10 @@ jobs:
       - name: Allowlist guard
         run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh
 
-  # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep and njsscan
-  # (pinned, hash-locked where the ecosystem allows; no advisory-DB download) and checks each is RAN with its rule id on one canary. trivy, bundler-audit, kubescape and retire need a downloaded DB or rule set, and cppcheck has no pinned, hash-verifiable binary, so they stay stub-only.
+  # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep, njsscan, tflint, kics, cppcheck and devskim
+  # (pinned, hash-locked where the ecosystem allows; no advisory-DB download) and checks each is RAN with its rule id on one canary. trivy, bundler-audit, kubescape and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran.
   real-hub:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04  # pinned: the cppcheck .deb below is the noble build and its runtime deps come from noble
     timeout-minutes: 15
     permissions:
       contents: read
@@ -626,6 +626,50 @@ jobs:
           curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/zegl/kube-score/releases/download/v1.20.0/kube-score_1.20.0_linux_amd64.tar.gz
           echo "ff777b4b3f45905b2d22c69e93c57a7a170e61529a79d54e59cb95cca4bc2a55  kube-score_1.20.0_linux_amd64.tar.gz" | sha256sum -c -
           tar -xzf kube-score_1.20.0_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" kube-score
+      - name: Install tflint 0.66.0 (pinned sha256 from the release checksums.txt; the bundled terraform ruleset, no plugin download)
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/bin"
+          cd "$RUNNER_TEMP"
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/terraform-linters/tflint/releases/download/v0.66.0/tflint_linux_amd64.zip
+          echo "22be11cadc25cd02401e94c0ccc91aa63467debb8485c2a755d98f4c0a9d1e2e  tflint_linux_amd64.zip" | sha256sum -c -
+          unzip -q -o tflint_linux_amd64.zip tflint -d "$RUNNER_TEMP/bin"
+      - name: Install kics 2.1.20 (pinned sha256 from the release checksums.txt; queries from the release's pinned commit)
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/bin"
+          cd "$RUNNER_TEMP"
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/Checkmarx/kics/releases/download/v2.1.20/kics_2.1.20_linux_amd64.tar.gz
+          echo "8a5aa375ccfdc0ddd1114eddf1f9638ad7f6122e98d12a592207509dbe6d81f8  kics_2.1.20_linux_amd64.tar.gz" | sha256sum -c -
+          tar -xzf kics_2.1.20_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" kics
+          # the release tarball ships no queries: fetch the v2.1.20 commit by its full sha (content-addressed) and check it
+          git init -q kics-src
+          git -C kics-src fetch -q --depth 1 https://github.com/Checkmarx/kics.git e1f23cad9640f55b963f22a116b04906b8c16ac6
+          test "$(git -C kics-src rev-parse FETCH_HEAD)" = e1f23cad9640f55b963f22a116b04906b8c16ac6
+          git -C kics-src checkout -q --detach FETCH_HEAD
+          echo "KICS_QUERIES_PATH=$RUNNER_TEMP/kics-src/assets/queries" >> "$GITHUB_ENV"
+          echo "KICS_LIBRARIES_PATH=$RUNNER_TEMP/kics-src/assets/libraries" >> "$GITHUB_ENV"
+      - name: Install cppcheck 2.13.0-2ubuntu3 (pinned .deb sha256 from Launchpad's published-binary record; runtime deps via apt)
+        run: |
+          set -euo pipefail
+          cd "$RUNNER_TEMP"
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://launchpad.net/ubuntu/+archive/primary/+files/cppcheck_2.13.0-2ubuntu3_amd64.deb
+          echo "c84251e17148b6b4ff3698bb71ceec0f445896d97888bedad8bba471c00aeadf  cppcheck_2.13.0-2ubuntu3_amd64.deb" | sha256sum -c -
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq --no-install-recommends "$RUNNER_TEMP/cppcheck_2.13.0-2ubuntu3_amd64.deb"
+      - name: Install DevSkim CLI 1.0.100 (nupkg pinned sha512 = the nuget.org catalog packageHash; installed from the local file)
+        env:
+          DOTNET_CLI_TELEMETRY_OPTOUT: "1"
+          DOTNET_NOLOGO: "1"
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/nupkg"
+          curl --proto '=https' --tlsv1.2 -fsSL -o "$RUNNER_TEMP/nupkg/microsoft.cst.devskim.cli.1.0.100.nupkg" https://api.nuget.org/v3-flatcontainer/microsoft.cst.devskim.cli/1.0.100/microsoft.cst.devskim.cli.1.0.100.nupkg
+          test "$(openssl dgst -sha512 -binary "$RUNNER_TEMP/nupkg/microsoft.cst.devskim.cli.1.0.100.nupkg" | base64 -w0)" = "WSAI8ReSrSF6BdM1ZWe2uupS7PrjSr9/1c0eor9kCPL6e6pt8t3OfCo8fqrWvzlutgSb7kbozg4wFPjvpqRTPA=="
+          # package signature check only where the runner's dotnet ships the verb
+          if dotnet nuget verify --help >/dev/null 2>&1; then dotnet nuget verify "$RUNNER_TEMP/nupkg/microsoft.cst.devskim.cli.1.0.100.nupkg"; fi
+          dotnet tool install Microsoft.CST.DevSkim.CLI --version 1.0.100 --tool-path "$RUNNER_TEMP/devskim" --add-source "$RUNNER_TEMP/nupkg"
+          echo "$RUNNER_TEMP/devskim" >> "$GITHUB_PATH"
       - name: Real-tool canaries
         run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh
 

changelog.d/hub-real-smoke-6.md

@@ -0,0 +1,3 @@
+### Changed
+- The real-hub CI job now also runs real tflint 0.66.0, kics 2.1.20, cppcheck 2.13.0-2ubuntu3 and DevSkim CLI 1.0.100 through `review_checks.sh`, each on a fictional canary file; the job fails unless the tool ran and reported its expected rule on that file with no not-run or NOT CHECKED line. Pins: tflint and kics by sha256 from each release `checksums.txt` (tflint uses its bundled terraform ruleset, no `--init`; the kics release tarball ships no queries, so the queries come from the release commit fetched by its full sha and verified); cppcheck as the noble `.deb` fetched from its stable Launchpad URL and checked by sha256, runtime deps from apt (the job now runs on `ubuntu-24.04` instead of `ubuntu-latest` to match); DevSkim's package signature is also checked with `dotnet nuget verify` where the runner's dotnet has it; DevSkim by the nupkg sha512 from the nuget.org catalog `packageHash`, installed from the downloaded file.
+- Not added: syft, an inventory the hub never counts as ran, so a canary could not assert a run.

scripts/test-review-checks-real-hub.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
-# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep (repo rules) and njsscan (no stubs) on one tiny
+# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep (repo rules) and njsscan, tflint, kics, cppcheck and devskim (no stubs) on one tiny
 # known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line)
 # with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure
-# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), cppcheck (no pinned, hash-verifiable binary), bundler-audit, kubescape and retire need a
+# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), bundler-audit, kubescape and retire need a
 # network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download.
 # A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs.
 # Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries.
@@ -11,7 +11,7 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd)
 RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh}
 fail=0; tools=
 bin() { [ "$1" = ruff-security ] && echo ruff || echo "$1"; }  # the hub tool name ruff-security runs the ruff binary
-for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score semgrep njsscan; do
+for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score semgrep njsscan tflint kics cppcheck devskim; do
   if command -v "$(bin "$t")" >/dev/null 2>&1; then tools+=" $t"
   elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1
   else echo "skip - $t not installed"; fi
@@ -22,7 +22,7 @@ trap 'cd /; rm -r -- "$T"' EXIT
 mkdir "$T/repo" && cd "$T/repo" || exit 1
 G() { git -c user.email=t@example.com -c user.name=t "$@"; }
 G init -q . && printf 'x\n' >README.md && G add . && G commit -q -m base && G branch base
-mkdir -p infra playbooks app docker .github/workflows contracts gocanary k8s
+mkdir -p tfl infra playbooks app docker .github/workflows contracts gocanary k8s
 printf 'resource "aws_s3_bucket" "acme" {\n  bucket = "acme-canary"\n}\n' >infra/main.tf
 printf -- '---\n- hosts: all\n  tasks:\n    - ansible.builtin.shell: echo hello\n' >playbooks/site.yml
 printf 'eval(params)\n' >app/canary.rb  # Security/Eval skips a plain string literal argument (cop source: pattern $!str), so the argument must not be one
@@ -42,19 +42,24 @@ tk=$(printf 'gh%s_%s' p a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8); printf 'token = %
 printf 'rules:\n  - id: acme-canary\n    languages: [generic]\n    severity: ERROR\n    message: fictional canary marker\n    pattern: ACME_SEMGREP_CANARY\n' >.semgrep.yml  # repo rules: the hub uses them instead of the registry pack, so no network
 printf 'ACME_SEMGREP_CANARY\n' >app/sg.txt
 printf "var express = require('express');\nvar app = express();\napp.get('/', function (req, res) {\n  eval(req.query.code);\n  res.send('ok');\n});\n" >app/canary.js  # njsscan eval_nodejs (ERROR)
+printf 'variable "acme_unused" {\n  type = string\n}\n' >tfl/main.tf  # tflint terraform_unused_declarations (bundled ruleset, recommended preset, no --init)
+printf 'void f(void) {\n  char a[4];\n  a[10] = 0;\n}\n' >app/cpp_canary.c  # cppcheck arrayIndexOutOfBounds (default checks)
+printf 'import hashlib\n\nh = hashlib.md5(b"acme")\n' >app/ds_canary.py  # devskim DS126858 weak hash
 G add . && G commit -q -m canaries
 rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?
 echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err"
 # tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids)
 want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;;
   pmd) echo '^(HardCodedCryptoKey|InsecureCryptoIv)$' ;; solhint) echo '^avoid-tx-origin$' ;; hadolint) echo '^DL3007$' ;; bandit) echo '^B602$' ;; ruff-security) echo '^S602$' ;; actionlint) echo '^syntax-check$' ;; zizmor) echo '^unpinned-uses$' ;; gosec) echo '^G(401|501)$' ;; staticcheck) echo '^SA4000$' ;;
   kube-linter) echo '^latest-tag$' ;; flawfinder) echo '^FF1013$' ;; detect-secrets) echo '^Basic Auth Credentials$' ;;
-  gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;; esac; }
+  gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;;
+  tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;; esac; }
 # tool -> its canary path (a row must be on this file, not just carry the rule id)
 cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;;
   pmd) echo app/Canary.java ;; solhint) echo contracts/Canary.sol ;; hadolint) echo docker/Dockerfile ;; bandit|ruff-security) echo app/canary.py ;; actionlint) echo .github/workflows/canary.yml ;; zizmor) echo .github/workflows/zcanary.yml ;; gosec|staticcheck) echo gocanary/canary.go ;;
   kube-linter) echo k8s/pod.yaml ;; flawfinder) echo app/canary.c ;; detect-secrets) echo app/notes.txt ;; gitleaks|trufflehog) echo app/token.cfg ;;
-  checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;; esac; }
+  checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;;
+  tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; esac; }
 # the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head
 raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;;
   rubocop) rubocop --force-default-config --fail-level fatal --format json --cache false -- ./app/canary.rb ;; cfn-lint) cfn-lint --format json -- ./infra/stack.yaml ;;
@@ -65,7 +70,9 @@ raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft
   kube-linter) kube-linter lint --format json ./k8s/pod.yaml ;; flawfinder) flawfinder --sarif --quiet ./app/canary.c ;; detect-secrets) detect-secrets scan --no-verify ./app/notes.txt ;;
   gitleaks) gitleaks detect --no-git --no-banner --exit-code 0 -f json -r /dev/stdout -s ./app/token.cfg ;; trufflehog) trufflehog filesystem --no-verification --no-update --sarif ./app/token.cfg ;;
   checkov) CKV_SKIP_PACKAGE_UPDATE_CHECK=true checkov -f ./docker/Dockerfile -o json --quiet --skip-download --soft-fail ;; kube-score) kube-score score -o json ./k8s/pod.yaml ;;
-  semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;; esac; }
+  semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;;
+  tflint) tflint --format=json --no-color --call-module-type=none --chdir=./tfl --filter=main.tf ;; kics) kics scan -p ./docker/Dockerfile --no-progress --disable-secrets --report-formats json -o "$T/kics-raw"; cat "$T/kics-raw/results.json" ;;
+  cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;; esac; }
 for t in $tools; do
   re=$(want "$t"); cf=$(cany "$t")
   if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \