Hub: drop nuclei info-severity rows (technology fingerprints)

.claude/skills/deep-code-review/scripts/review_checks.sh

@@ -1171,7 +1171,8 @@ try:
     elif k == "nuclei":  # JSONL: one finding per line; a non-JSON line is a failure, not noise
         rows = []
         for ln in filter(None, (x.strip() for x in t.splitlines())):
-            d = json.loads(ln); rows.append((d["matched-at"], 0, d["template-id"], f"{d['info']['severity']}: {d['info']['name']}"))
+            d = json.loads(ln)
+            if d["info"]["severity"] != "info": rows.append((d["matched-at"], 0, d["template-id"], f"{d['info']['severity']}: {d['info']['name']}"))  # info = technology fingerprint, not a finding
     else:  # zap
         rows = [(i.get("uri", "."), 0, "zap-" + a["pluginid"], f"{a['riskdesc']}: {a['alert']}") for s in json.loads(t)["site"] for a in s["alerts"] for i in (a.get("instances") or [{}])[:3]]
 except Exception:

.github/workflows/ci.yml

@@ -519,7 +519,7 @@ jobs:
   # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck, grype, trivy and osv-scanner, which fetch their vulnerability DB online: trivy by pinned digest, the rest live) and checks each is RAN with its rule id on one canary. bundler-audit and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran.
   real-hub:
     runs-on: ubuntu-24.04  # pinned: the cppcheck .deb below is the noble build and its runtime deps come from noble
-    timeout-minutes: 25
+    timeout-minutes: 35
     permissions:
       contents: read
     steps:
@@ -821,6 +821,18 @@ jobs:
           chmod +x "$RUNNER_TEMP/bin/mobsfscan"
           echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
           "$RUNNER_TEMP/venv-mobsfscan/bin/python" -c "import importlib.metadata as m; assert m.version('semgrep') == '1.172.0', m.version('semgrep')"
+      - name: Install nuclei 3.11.1 and nuclei-templates v10.5.2 (pinned sha256; the binary hash is the upstream checksums-file entry, the templates hash is of the GitHub tag archive)
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/bin"
+          cd "$RUNNER_TEMP"
+          curl --proto '=https' --tlsv1.2 -fsSL --retry 3 --max-time 300 -O https://github.com/projectdiscovery/nuclei/releases/download/v3.11.1/nuclei_3.11.1_linux_amd64.zip
+          echo "ea63d4ae232808cd7c6bc00d0142428e231fab59dae01042246097d195835ab6  nuclei_3.11.1_linux_amd64.zip" | sha256sum -c -
+          unzip -q nuclei_3.11.1_linux_amd64.zip nuclei -d "$RUNNER_TEMP/bin"
+          curl --proto '=https' --tlsv1.2 -fsSL --retry 3 --max-time 300 -o nuclei-templates.tar.gz https://github.com/projectdiscovery/nuclei-templates/archive/refs/tags/v10.5.2.tar.gz
+          echo "016b7abbf931d4d1a851a5a568e44510dc61691dc8a5202e3b24e701515ec574  nuclei-templates.tar.gz" | sha256sum -c -
+          mkdir -p "$HOME/nuclei-templates"
+          tar -xzf nuclei-templates.tar.gz --strip-components=1 -C "$HOME/nuclei-templates"
       - name: Real-tool canaries
         run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh
 

changelog.d/hub-real-smoke-13.md

@@ -0,0 +1,2 @@
+### Changed
+- The real-hub CI job now runs the real nuclei (3.11.1, sha256 from the upstream checksums file) with nuclei-templates v10.5.2 (tag archive, sha256-pinned) through the hub's DAST route (REVIEW_DAST_URL) against a local python http.server canary, and asserts it ran and reported the exact rule id tech-detect with no "not run" or "NOT CHECKED" line. zap-baseline is deliberately not covered: its only route is the ghcr.io/zaproxy/zaproxy image, about 1.2 GB of compressed layers, too heavy for the per-push job.

scripts/test-review-checks-posture-dast.sh

@@ -28,7 +28,8 @@ stub nu nuclei '[ "${HUB_MODE:-}" != crash ] || exit 9
 echo "gt=${GITHUB_TOKEN:-}" >>"$HUB_LOG"
 case " $* " in *" -tl "*) printf "%s" "${TL_OUT-tpl1}"; exit 0 ;; esac
 [ -z "${EMPTY_SCAN:-}" ] || exit 0
-printf "%s\n" "{\"template-id\":\"missing-xfo\",\"info\":{\"name\":\"X-Frame-Options missing\",\"severity\":\"low\"},\"matched-at\":\"http://localhost:3000/\"}"'
+printf "%s\n" "{\"template-id\":\"missing-xfo\",\"info\":{\"name\":\"X-Frame-Options missing\",\"severity\":\"low\"},\"matched-at\":\"http://localhost:3000/\"}"
+printf "%s\n" "{\"template-id\":\"tech-detect\",\"info\":{\"name\":\"Wappalyzer\",\"severity\":\"info\"},\"matched-at\":\"http://localhost:3000/\"}"'
 stub sg semgrep 'printf "%s\n" "{\"results\":[],\"errors\":[],\"paths\":{\"scanned\":[\"a.py\"]}}"'
 stub zap zap-baseline.py '[ "${HUB_MODE:-}" != crash ] || exit 9
 while [ $# -gt 0 ]; do [ "$1" != -J ] || j=$2; shift; done
@@ -96,7 +97,7 @@ CURL_FAIL=1 REVIEW_DAST_URL=http://localhost:3000/ run "$T/nu:$T/zap"; [ ! -s "$
 TL_OUT= EMPTY_SCAN=1 REVIEW_DAST_URL=http://localhost:3000/ run "$T/nu"; grep -q '^NOT CHECKED: nuclei (no templates matched' "$T/err" && row '.tool=="nuclei" and .rule=="not-checked"' && ! ran nuclei && ok "empty nuclei output, no template count: not-checked" || bad "empty nuclei"
 EMPTY_SCAN=1 REVIEW_DAST_URL=http://localhost:3000/ run "$T/nu"; ran nuclei && ok "templates matched, no findings: ran clean" || bad "clean nuclei"
 GITHUB_TOKEN=tok REVIEW_DAST_URL=http://localhost:3000/ run "$T/nu:$T/zap"
-row '.tool=="nuclei" and .rule=="missing-xfo" and .file=="http://localhost:3000/"' && row '.tool=="zap" and .rule=="zap-10038"' && ran nuclei && ran zap && grep -q -- '-s info,low -type http -tags tech,exposure -etags default-login,xss,ssrf,oast,fuzz,fuzzing,takeover,cve,intrusive,dos' "$HUB_LOG" && grep -q -- ' -ni ' "$HUB_LOG" && ! grep -q 'gt=tok' "$HUB_LOG" && ok "nuclei and zap findings parsed, allowlist and exclusions, no interactsh, no GitHub token" || bad "DAST findings"
+row '.tool=="nuclei" and .rule=="missing-xfo" and .file=="http://localhost:3000/"' && ! row '.tool=="nuclei" and .rule=="tech-detect"' && row '.tool=="zap" and .rule=="zap-10038"' && ran nuclei && ran zap && grep -q -- '-s info,low -type http -tags tech,exposure -etags default-login,xss,ssrf,oast,fuzz,fuzzing,takeover,cve,intrusive,dos' "$HUB_LOG" && grep -q -- ' -ni ' "$HUB_LOG" && ! grep -q 'gt=tok' "$HUB_LOG" && ok "nuclei and zap findings parsed, allowlist and exclusions, no interactsh, no GitHub token" || bad "DAST findings"
 grep -q -- "-g --noproxy \* -sf" "$T/curl.log" && ok "probe: curl -g --noproxy '*'" || bad "curl flags"
 mkdir -p "$T/dk"; stub dk docker ':'
 REVIEW_DAST_URL=http://localhost:3000/ run "$T/dk"; grep -q '^docker run --rm --pull=never ' "$HUB_LOG" && ok "ZAP docker run never auto-pulls the image" || bad "docker pull"

scripts/test-review-checks-real-hub.sh

@@ -4,21 +4,22 @@
 # with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure
 # handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), bundler-audit and retire need a
 # network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download. codeql is dropped: its CLI bundle is a multi-hundred-MB download plus per-language database builds, too heavy for this per-push job. psalm is dropped: the hub's row builder reads the is_security issue field, which psalm 6.x does not emit and psalm 7.x does, and psalm 7.x's platform check needs PHP 8.3.16 or newer while the runner ships 8.3.6, so it would need a third-party PHP setup action and cannot be run here. sobelow is dropped: it needs an Erlang/OTP and Elixir toolchain the runner does not ship, with no checksum-pinned install path. licensee is dropped: the hub reports it as an inventory line, never as a scan that ran, and its dependency set includes natively built gems (rugged) that cannot be hash-locked without a Bundler lockfile. pip-audit is dropped: the hub reaches it only as a fallback for a requirements*.txt that osv-scanner and trivy gave no result for (the seeded osv DB covers requirements.txt, so it would never run here), and it has no offline advisory DB: it queries PyPI live, so there is nothing to integrity-pin.
+# nuclei (the hub's DAST route) scans a local python http.server canary through REVIEW_DAST_URL; its rule id is ds-store-file (a low-severity exposure: the canary serves a .DS_Store; the hub drops info-severity fingerprints such as tech-detect). zap-baseline is dropped: the only route is the ghcr.io/zaproxy/zaproxy image, whose compressed layers total about 1.2 GB (registry manifest, checked 2026-11-13), over the per-push job's size budget.
 # A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs.
 # Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries.
 set -euo pipefail
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh}
 fail=0; tools=
 bin() { case $1 in ruff-security) echo ruff ;; eslint-security) echo eslint ;; *) echo "$1" ;; esac; }  # the hub tool names ruff-security and eslint-security run the ruff and eslint binaries
-for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score kubescape semgrep njsscan tflint kics cppcheck devskim govulncheck grype trivy osv-scanner shellcheck guarddog cargo-audit cargo-deny brakeman mobsfscan eslint-security; do
+for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score kubescape semgrep njsscan tflint kics cppcheck devskim govulncheck grype trivy osv-scanner shellcheck guarddog cargo-audit cargo-deny brakeman mobsfscan eslint-security nuclei; do
   if command -v "$(bin "$t")" >/dev/null 2>&1; then tools+=" $t"
   elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1
   else echo "skip - $t not installed"; fi
 done
 [ -n "$tools" ] || { [ "$fail" = 0 ] && echo "no real hub tool installed: SKIP test-review-checks-real-hub"; exit "$fail"; }
 T=$(mktemp -d "${TMPDIR:-/tmp}/realhub.XXXXXX") && [ -d "$T" ] || { echo "FAIL - mktemp" >&2; exit 1; }
-trap 'cd /; rm -r -- "$T"' EXIT
+SP=; trap '[ -z "$SP" ] || kill "$SP" 2>/dev/null; cd /; rm -r -- "$T"' EXIT
 mkdir "$T/repo" && cd "$T/repo" || exit 1
 G() { git -c user.email=t@example.com -c user.name=t "$@"; }
 G init -q . && # eslint-security: the project's own flat config (committed in the base, because the hub does not run a config the diff changes) loading the pinned security plugin
@@ -80,14 +81,22 @@ case " $tools " in *" eslint-security "*)
 # osv-scanner scans the lockfile above (lodash 4.17.15) against the offline DB the CI step seeds into REVIEW_OSV_DB_DIR (the hub's own warm command). guarddog scans the package dir the lock newly adds, so the lock's one package is installed here (untracked, after the commit) with a fictional mining-pool payload: threat-process-cryptomining
 case " $tools " in *" guarddog "*) mkdir -p node_modules/lodash && printf '{"name":"lodash","version":"4.17.15"}\n' >node_modules/lodash/package.json && printf '// fictional canary\nvar pool = "stratum+tcp://pool.example.com:3333"; // xmrig\n' >node_modules/lodash/index.js ;; esac
 # govulncheck (vuln.go.dev), grype, trivy, osv-scanner (DB seed), guarddog, cargo-audit, cargo-deny, brakeman, mobsfscan and eslint-security (a transient crash reads as not run) are online or DB-backed: retry the hub run up to 3 times while any is not run
-for i in 1 2 3; do rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?; grep -qE 'not run:.*\b(govulncheck|grype|trivy|osv-scanner|guarddog|cargo-audit|cargo-deny|brakeman|mobsfscan|eslint-security)\b' "$T/err" && [ "$i" -lt 3 ] || break; sleep $((i * 10)); done
+# nuclei: a localhost python http.server canary (outside the repo) on a free port; its default template pass is rate-limited (~3 min), so the hub's per-tool --timeout is raised
+tmo=60
+case " $tools " in *" nuclei "*)
+  mkdir "$T/site" && printf '<h1>acme canary</h1>\n' >"$T/site/index.html" && printf '\0\0\0\1Bud1\0' >"$T/site/.DS_Store"
+  port=$(python3 -I -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
+  python3 -I -m http.server "$port" --bind 127.0.0.1 --directory "$T/site" >/dev/null 2>&1 & SP=$!
+  export REVIEW_DAST_URL="http://127.0.0.1:$port" && tmo=420
+  for _ in 1 2 3 4 5 6 7 8 9 10; do curl -sf --max-time 2 -o /dev/null "$REVIEW_DAST_URL" && break; sleep 1; done ;; esac
+for i in 1 2 3; do rc=0; bash "$RC" --base base --timeout "$tmo" >"$T/out" 2>"$T/err" || rc=$?; grep -qE 'not run:.*\b(govulncheck|grype|trivy|osv-scanner|guarddog|cargo-audit|cargo-deny|brakeman|mobsfscan|eslint-security|nuclei)\b' "$T/err" && [ "$i" -lt 3 ] || break; sleep $((i * 10)); done
 echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err"
 # tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids)
 want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;;
   pmd) echo '^(HardCodedCryptoKey|InsecureCryptoIv)$' ;; solhint) echo '^avoid-tx-origin$' ;; hadolint) echo '^DL3007$' ;; bandit) echo '^B602$' ;; ruff-security) echo '^S602$' ;; actionlint) echo '^syntax-check$' ;; zizmor) echo '^unpinned-uses$' ;; gosec) echo '^G(401|501)$' ;; staticcheck) echo '^SA4000$' ;;
   kube-linter) echo '^latest-tag$' ;; flawfinder) echo '^FF1013$' ;; detect-secrets) echo '^Basic Auth Credentials$' ;;
   gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; kubescape) echo '^C-0057$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;;
-  tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;;
+  tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;; nuclei) echo '^ds-store-file$' ;;
   govulncheck) echo '^GO-(2020-0036|2021-0061|2022-0956)$' ;; grype) echo '^(GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw)|CVE-2021-23337|CVE-2020-8203)$' ;; trivy) echo '^DS-000[12]$' ;;
   osv-scanner) echo '^GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw|29mw-wpgm-hmr9|r5fr-rjxr-66jc|xxjr-mmjv-4gpg)$' ;; shellcheck) echo '^SC2086$' ;; guarddog) echo '^threat-process-cryptomining$' ;;
   cargo-audit) echo '^RUSTSEC-2020-0071$' ;; cargo-deny) echo '^rejected$' ;; brakeman) echo '^Evaluation$' ;; mobsfscan) echo '^android_manifest_debugging_enabled$' ;; eslint-security) echo '^security/detect-eval-with-expression$' ;; esac; }
@@ -96,7 +105,7 @@ cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/si
   pmd) echo app/Canary.java ;; solhint) echo contracts/Canary.sol ;; hadolint) echo docker/Dockerfile ;; bandit|ruff-security) echo app/canary.py ;; actionlint) echo .github/workflows/canary.yml ;; zizmor) echo .github/workflows/zcanary.yml ;; gosec|staticcheck) echo gocanary/canary.go ;;
   kube-linter) echo k8s/pod.yaml ;; flawfinder) echo app/canary.c ;; detect-secrets) echo app/notes.txt ;; gitleaks|trufflehog) echo app/token.cfg ;;
   checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; kubescape) echo k8s/ks.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;;
-  tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; trivy) echo docker/Dockerfile ;; osv-scanner) echo package-lock.json ;; shellcheck) echo app/canary.sh ;; guarddog) echo node_modules/lodash/index.js ;;
+  tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; nuclei) echo "" ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; trivy) echo docker/Dockerfile ;; osv-scanner) echo package-lock.json ;; shellcheck) echo app/canary.sh ;; guarddog) echo node_modules/lodash/index.js ;;
   cargo-audit|cargo-deny) echo Cargo.lock ;; brakeman) echo app/controllers/canary_controller.rb ;; mobsfscan) echo AndroidManifest.xml ;; eslint-security) echo app/canary.js ;; esac; }
 # the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head
 raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;;
@@ -110,7 +119,7 @@ raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft
   checkov) CKV_SKIP_PACKAGE_UPDATE_CHECK=true checkov -f ./docker/Dockerfile -o json --quiet --skip-download --soft-fail ;; kube-score) kube-score score -o json ./k8s/pod.yaml ;; kubescape) kubescape scan framework nsa,cis-v1.23-t1.0.1 ./k8s/ks.yaml --format json --output "$T/ks-raw.json" --keep-local --logger error --use-artifacts-from "${KUBESCAPE_ARTIFACTS:-$HOME/.kubescape}" >/dev/null && cat "$T/ks-raw.json" ;;
   semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;;
   tflint) tflint --format=json --no-color --call-module-type=none --chdir=./tfl --filter=main.tf ;; kics) kics scan -p ./docker/Dockerfile --no-progress --disable-secrets --report-formats json -o "$T/kics-raw"; cat "$T/kics-raw/results.json" ;;
-  cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;;
+  cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;; nuclei) nuclei -u "$REVIEW_DAST_URL" -jsonl -silent -nc -ni -duc -rl 10 -s info,low -type http -tags tech,exposure ;;
   govulncheck) GOPROXY=off GOTOOLCHAIN=local govulncheck -format json ./... ;; grype) GRYPE_DB_AUTO_UPDATE=false GRYPE_CHECK_FOR_APP_UPDATE=false grype sbom:./sbom/bom.json -o json ;;
   trivy) trivy ${REVIEW_TRIVY_CACHE:+--cache-dir "$REVIEW_TRIVY_CACHE"} fs --quiet --format json --skip-version-check --skip-db-update --offline-scan --scanners vuln,misconfig . ;;
   osv-scanner) OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY="${REVIEW_OSV_DB_DIR:-}" osv-scanner --offline-vulnerabilities --format json -L ./package-lock.json ;; shellcheck) shellcheck -f gcc ./app/canary.sh ;;