diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b79c3c5..372f3d3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,4 +7,4 @@ ## Testing --- -Reviewed against [the review checklist](https://github.com/WireCatLabs/cli-messaging/blob/main/docs/dev/REVIEW.md) — §A for a docs PR, §B for code. +Reviewed against [the review checklist](https://github.com/WireCatLabs/cli-core/blob/main/docs/dev/REVIEW.md) — §A for a docs PR, §B for code. diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 0000000..a26a415 --- /dev/null +++ b/.semgrepignore @@ -0,0 +1,26 @@ +# A .semgrepignore replaces Semgrep's built-in list, so that list (1.178.0) is repeated here. +.git +.svn +.hg +_darcs +CVS +build/ +vendor/ +dist/ +*.min.js +.env/ +.tox/ +node_modules/ +.npm/ +.yarn/ +.venv/ +_opam/ +_build/ +_cargo/ +test/ +tests/ +testsuite/ +*_test.go + +# Saved third-party web pages for the reader benchmarks; not code, and they time the scanner out. +performance/**/fixtures/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2666701..48c7aae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,21 @@ Every entry says what changed as a caller sees it, why, and what to watch for discovery contracts are refused. The research report distinguishes released behavior from newer source changes and includes a separately authorized live Google checklist. +### Fixed + +- `cli-testing security scan` runs its Semgrep stage from an installed package: the stage ships as + compiled `dist/semgrep.js`, because Node refuses to strip types under `node_modules`. Before, every + scan from an install failed at Semgrep; it worked only from a checkout. +- A repository whose scanner fails no longer stops the run: `scan` records the failure, scans the + remaining repositories and exits 1 at the end, naming the failed ones on stderr. Watch for a + non-zero exit with most raw files present — the failed repository has none for that tool. +- The Semgrep fallback covers JavaScript files the parser rejects (`.js`, `.jsx`, `.mjs`, `.cjs`, + plus `.mts`, `.cts`), and keeps top-level await. One rule failing on one file is recorded as a + coverage gap in `semgrep.json` and `SCOPE.md` instead of failing the repository; every other + scanner error still does. +- This repository's saved third-party web pages under `performance/**/fixtures/` are excluded from + Semgrep in `.semgrepignore`, so they no longer time the scan out. + ## 0.4.0 — 11.10.2026 ### Changed — may break callers diff --git a/CLAUDE.md b/CLAUDE.md index 61c07f9..1b47618 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,8 +5,9 @@ as a development dependency. Start with the one page that covers what you are ab - [`docs/dev/ARCHITECTURE.md`](docs/dev/ARCHITECTURE.md) — the suite groups and how a CLI plugs in. - [`docs/dev/BACKLOG.md`](docs/dev/BACKLOG.md) — what comes next, in order. -- [`docs/security/`](docs/security/METHOD.md) — how a security audit is run and repeated; its - scripts are in `scripts/security/` and `scripts/agent/`. +- [`docs/security/`](docs/security/METHOD.md) — how a security audit of every WireCat repository is + run and repeated; its scripts are in `scripts/security/` and `scripts/agent/`. Vulnerabilities are + reported as the [security policy](https://github.com/WireCatLabs/.github/blob/main/SECURITY.md) says. - [`docs/dev/CONVENTIONS.md`](docs/dev/CONVENTIONS.md) — the shared conventions, and what differs here. - [`docs/dev/TESTING.md`](docs/dev/TESTING.md) — the checks and the coverage floor. - [`docs/dev/agents.md`](docs/dev/agents.md) — what an agent may change here, and what stops it. @@ -16,13 +17,14 @@ as a development dependency. Start with the one page that covers what you are ab 1. **A suite drives a CLI from the outside.** Its binary, its MCP server, its published package — never its source. What a CLI must supply (its MCP command, how the test account sends and deletes, which chats are test chats) is config in that CLI's repository. `biome.json` refuses - messenger libraries and cli-messaging under `src/`. + messenger libraries and cli-core under `src/`. 2. **Nothing touches a real account on its own.** `live` and the agent suite run only when the owner starts them, on the test profiles and test chats of a private cast that is never committed. 3. **Public tools, private results.** Run results, the findings register and the cast live in the private `cli-private` repository. Nothing here names an unfixed vulnerability. -4. **Speed of development comes first.** No suite runs on a pull request or a commit: on a release, - on demand or on a schedule only. +4. **Speed of development comes first.** No suite runs on a pull request or a local commit: on a + release, on demand or on a schedule only. This repository's own tests run once on each merge to + `main`. 5. **No message, token or phone number in a fixture, an output file or a document.** Seed data is synthetic. 6. **Every external tool is pinned** — a version, and a checksum where it is a downloaded binary — @@ -60,6 +62,7 @@ A branch off `main`, in a worktree, and a pull request. A change a caller can se Local commits check only staged lint and secrets. There is no pre-push check. Config integrity, repository lint, Markdown, and secret detection run in PR CI. The bounded `ci:quick` step adds typechecking for CLI packages or synthetic unit tests for the docs site. -Full tests, coverage, builds, parity, browser and platform suites run for releases, -explicit manual validation, and monthly nonpublishing validation. See the +Full tests, coverage and builds also run once on each merge to `main` (Linux only, no publishing). +Browser and platform suites run for releases, explicit manual validation, and monthly nonpublishing +validation. See the [shared policy](https://github.com/WireCatLabs/community/blob/main/standards/README.md#ci-and-hooks). diff --git a/docs/ci/MONTHLY.md b/docs/ci/MONTHLY.md index be6bd62..bab7e33 100644 --- a/docs/ci/MONTHLY.md +++ b/docs/ci/MONTHLY.md @@ -13,15 +13,15 @@ and CodeQL. Messaging includes C/C++ analysis as well as JavaScript/TypeScript a | --- | --- | | cli-base / monthly.yml | 02:17 | | cli-memo / monthly.yml | 02:31 | -| cli-messaging / monthly.yml | 02:38 | +| cli-core / monthly.yml | 02:38 | | drive-cli / monthly.yml | 02:45 | | cli-testing / monthly.yml | 02:52 | | max-cli / monthly.yml | 02:59 | | tg-cli / monthly.yml | 03:06 | | zoom-cli / monthly.yml | 03:13 | | cli-docs / monthly.yml | 03:20 | -| cli-messaging / onnx.yml | 03:27 | -| cli-messaging / sqlite.yml | 03:37 | +| cli-core / onnx.yml | 03:27 | +| cli-core / sqlite.yml | 03:37 | | community / security-release.yml | 03:47 | GitHub runs schedules on the latest default-branch commit. Times are staggered away from the @@ -49,8 +49,8 @@ gh run list --repo WireCatLabs/cli-testing --workflow monthly.yml --limit 5 Native probes use their individual workflows with publication disabled: ```sh -gh workflow run onnx.yml --repo WireCatLabs/cli-messaging --ref main -f publish=false -gh workflow run sqlite.yml --repo WireCatLabs/cli-messaging --ref main -f publish=false +gh workflow run onnx.yml --repo WireCatLabs/cli-core --ref main -f publish=false +gh workflow run sqlite.yml --repo WireCatLabs/cli-core --ref main -f publish=false ``` Community security can be started from its **Security release** workflow without publishing. diff --git a/docs/security/LESSONS.md b/docs/security/LESSONS.md index 1f3550a..f08cf65 100644 --- a/docs/security/LESSONS.md +++ b/docs/security/LESSONS.md @@ -1,6 +1,6 @@ # What the tools taught us -Facts learned during the 2026-10-09 audit, with their source. Each cost time once. +Facts learned during the audits, with their source. Each cost time once. ## Scanners @@ -18,6 +18,15 @@ Facts learned during the 2026-10-09 audit, with their source. Each cost time onc `semgrep/mcp/hooks/post_tool.py`, read in semgrep 1.180.0). The CLI with `--metrics=off` avoids it. - **The Socket CLI prints the start of the API token in its banner.** `--no-banner` hides it, and it must come after the command — before it, the flag is not parsed. +- **A `.semgrepignore` replaces Semgrep's built-in ignore list** instead of adding to it: with a + one-line file, `test/`, `dist/` and `node_modules/` are scanned again (ran on semgrep 1.178.0). A + repository's file repeats the built-in list before its own lines. +- **gitleaks reads `.gitleaksignore` from the current folder**, not from the repository it scans. + Run it from the repository's root, or a scan from elsewhere reports the ignored hits again. +- **CodeQL 2.27 SARIF keeps the rules in `tool.extensions[].rules`**, and `tool.driver.rules` is + empty. A result points at its rule through `rule.toolComponent.index` and `rule.index`; a gate + that looks rules up in `driver.rules` by `ruleId` finds none and never fails. Resolve every + result to a rule, and fail when one has no rule. - **Trail of Bits `supply-chain-risk-auditor` does not read `pnpm-lock.yaml`**; it covers direct dependencies only (its `SKILL.md`). osv-scanner covers the tree. - **Trail of Bits `fp-check` installs a Stop hook on every session** that blocks stopping until the @@ -33,6 +42,12 @@ Facts learned during the 2026-10-09 audit, with their source. Each cost time onc - **zizmor runs against `.github`**; given a nested path it can exit 3, "no inputs collected". - **Restricting the `npm` environment to `main`** keeps trusted publishing working from `main` (token exchange `201`) and stops branch dry runs from reaching npm. +- **A new repository does not inherit settings applied repository by repository.** Secret + scanning, push protection, security updates and private reporting start off unless an + organisation code security configuration is the default for new repositories. +- **CodeQL default setup and workflow CodeQL do not mix**: with default setup on, GitHub refuses + the SARIF a workflow uploads for the same languages + ([docs](https://docs.github.com/en/code-security/code-scanning/troubleshooting-sarif-uploads/default-setup-enabled)). - **A CodeQL alert number is per repository**; check every dismissed alert's path after dismissing by number. diff --git a/docs/security/METHOD.md b/docs/security/METHOD.md index d6fce8b..d92d2c8 100644 --- a/docs/security/METHOD.md +++ b/docs/security/METHOD.md @@ -7,24 +7,36 @@ folder the caller names — for the owner's CLIs, `security/runs//` `security/FINDINGS.md`. Step-by-step commands: [`RUNBOOK.md`](RUNBOOK.md). What the tools taught us: [`LESSONS.md`](LESSONS.md). -Repositories: `max-cli`, `tg-cli`, `cli-messaging` (with `packages/sqlite`, `packages/onnx`), -`cli-base`, `cli-testing`. All public. The private repository is never given to a -hosted scanner. +Repositories: every WireCatLabs repository — `max-cli`, `tg-cli`, `cli-core` (the shared messaging +core, with `packages/sqlite` and `packages/onnx`), `cli-base`, `cli-memo`, `drive-cli`, `zoom-cli`, +`cli-docs`, `cli-testing`, `community` and `.github`, all public. The private `cli-private` is +checked with gitleaks only and is never given to a hosted scanner. A repository left out of a run is +named in `SCOPE.md` with the reason, so a run never claims more coverage than it had. Check the list +against the organisation before each run: + +```sh +gh repo list WireCatLabs --limit 100 --json name,visibility --jq '.[] | "\(.name) \(.visibility)"' +``` ## A run, step by step -1. `mkdir runs/` and record what is scanned in `runs//SCOPE.md`: the commit of each - repository, and the version of every tool below. Without this, two runs cannot be compared. -2. Read back GitHub's settings for each repository (commands in "GitHub settings") into `SCOPE.md`. -3. Run [`scripts/security/scan`](../../scripts/security/scan) ` ...`. It exports - `origin/main` of each repository, runs the scanners, pulls open CodeQL alerts, strips temp paths, - and records each tool's exit code in `SCOPE.md`. A tool exit code its docs do not list as a - result fails the run, so a crash never reads as "no findings". -4. Run [`scripts/security/socket`](../../scripts/security/socket) ``. It scans the same - commits through Socket and adds its exit codes to `SCOPE.md`. The Socket CLI banner prints the - start of the API token; the script passes `--no-banner` after the command (before it, the flag is - not parsed). Raw output goes to `runs//raw//.*`, in JSON or SARIF where the - tool can produce it, so two runs can be diffed. +1. `cli-testing security scan ... [--previous ]` creates the run folder, + refuses tool versions other than the pins (or pins younger than two weeks), and calls + [`scripts/security/scan`](../../scripts/security/scan). The script exports `origin/main` of each + repository, writes `SCOPE.md` with the commit of each repository, the tool versions and the + GitHub settings it reads back, runs the scanners, pulls open CodeQL alerts, strips temp paths and + records each tool's exit code. A tool exit code its docs do not list as a result marks that + repository failed, so a crash never reads as "no findings"; the other repositories are still + scanned, and the run exits 1 at the end. `run.json` records the versions, exit codes, coverage and, + with `--previous`, the history needed to recognise a returning observation. +2. `cli-testing security socket ...` scans the same commits through Socket and + adds its exit code to `run.json`. The Socket CLI banner prints the start of the API token; the + script passes `--no-banner` after the command (before it, the flag is not parsed). Raw output + goes to `/raw//.json`, so two runs can be diffed. +3. gitleaks over the full history of every repository, `cli-private` included, from each + repository's root (it reads `.gitleaksignore` from the current folder): + `gitleaks git --no-banner --redact .`. Exit codes and hit counts go into `SCOPE.md`. +4. Read the GitHub settings that `scan` does not (commands in "GitHub settings") into `SCOPE.md`. 5. Do the manual review (section "Checklist"). Every item gets a verdict in `runs//REPORT.md`: safe, finding, or not applicable — each with the evidence that supports it. 6. Before a candidate finding gets an id, verify it against the current source and a bounded @@ -45,7 +57,7 @@ Skills and plugins (Claude Code): - Semgrep Guardian is **installed but disabled**. Even the local variant (`semgrep/guardian-local`) needs a Semgrep login, and its edit hook sends traces with the git remote, scanned paths and findings to Semgrep (`send_metrics=True` in `semgrep/mcp/hooks/post_tool.py`, semgrep 1.180.0). - The `semgrep` CLI with `--metrics=off` in `scripts/security/scan` covers the same rule packs. + The `semgrep` CLI with `--metrics=off` (set in `src/semgrep.ts`) covers the same rule packs. - The review prompt of `anthropics/claude-code-security-review` (`.claude/commands/security-review.md` in that repository), run locally through Claude Code. Not in CI. @@ -60,18 +72,35 @@ Command-line scanners, each pinned to a version recorded in `SCOPE.md`: - `semgrep` CE, pinned to a release at least two weeks old (1.178.0 on 2026-10-09; run 1 used 1.180.0, which was too new) — rule packs `p/javascript`, `p/typescript`, `p/nodejs`, `p/secrets`, `p/github-actions`. - When its pinned TypeScript parser fails on a whole file, the runner scans that file’s runtime - JavaScript compiled with `esbuild@0.25.12` targeting ES2019. It maps findings back to source and - keeps the original report, emitted code and source maps under `supplement/`. Compilation, runtime - parsing or missing coverage fails the run. Type-only syntax remains outside this fallback; - partial parsing warnings remain visible. + When its pinned parser fails on a whole TypeScript or JavaScript file (`.ts`, `.tsx`, `.mts`, + `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`), the runner scans that file’s runtime JavaScript compiled + with `esbuild@0.25.12` targeting ES2019, with top-level await kept. It maps findings back to source + and keeps the original report, emitted code and source maps under `supplement/`. Compilation, + runtime parsing or missing coverage fails that repository. Type-only syntax remains outside this + fallback; partial parsing warnings remain visible. One rule failing on one file (an `Internal + matching error` at level `warn`) is recorded as a gap in `semgrep.json` and `SCOPE.md`; any other + scanner error fails the repository. A repository's own `.semgrepignore` replaces Semgrep's + built-in ignore list, so it repeats that list; `scan` writes its patterns into `SCOPE.md`. - `zizmor` — GitHub Actions workflows. -- CodeQL — through GitHub default setup; alerts read with `gh api`. - -Release gates in the shared `release-checks.yml` (cli-base), required before each release build: `pnpm audit --prod --audit-level high` and -zizmor `--min-severity high`. A high advisory with no fix yet is let through per repository with -`audit.ignore` in `pnpm-workspace.yaml` (GHSA ids only — [pnpm audit docs](https://pnpm.io/cli/audit)). -pnpm has no field for a reason or an expiry, so every entry carries a comment with both: +- `gitleaks` — secrets in the full history of every repository. +- CodeQL — in the shared `release-checks.yml` (cli-base), which uploads its SARIF on `main`; open + alerts read with `gh api`. GitHub's CodeQL default setup stays off: it would take over the alerts + the workflow uploads. + +Release gates in the shared `release-checks.yml`, run before each release build (for cli-docs, before +the production deploy; a repository without a package release runs it from its manual +`security-release.yml`): + +- `pnpm audit --prod --audit-level critical` blocks a critical advisory on a production path. +- CodeQL blocks a finding whose rule has `security-severity` 9.0 or more. +- gitleaks over the full history blocks any leak. +- zizmor `--min-severity high` reports workflow findings with `--no-exit-codes`; it fails the gate + only when the scanner itself fails. + +High, medium and low findings from any tool go to the owner for review; they do not block a release. +A critical advisory with no fix yet is let through per repository with `audit.ignore` in +`pnpm-workspace.yaml` (GHSA ids only — [pnpm audit docs](https://pnpm.io/cli/audit)). pnpm has no +field for a reason or an expiry, so every entry carries a comment with both: ```yaml audit: @@ -88,33 +117,54 @@ gh api repos// --jq .security_and_analysis gh api repos///code-scanning/default-setup --jq .state gh api -i repos///vulnerability-alerts | head -1 # 204 = on gh api repos///automated-security-fixes --jq .enabled +gh api repos///private-vulnerability-reporting --jq .enabled +gh api repos///code-security-configuration --jq '"\(.status) \(.configuration.name)"' +gh api repos///environments/npm/deployment-branch-policies --jq '.branch_policies[].name' +gh api repos///rulesets --jq '.[].name' ``` -Expected: secret scanning, push protection, vulnerability alerts, Dependabot security updates on; -CodeQL `configured`. +Expected: secret scanning, push protection, vulnerability alerts, Dependabot security updates and +private vulnerability reporting on; CodeQL default setup `not-configured`; the organisation +configuration `WireCat public` attached; the `npm` environment, where the repository publishes, +limited to `main`; a ruleset on `main`. `cli-private` has no hosted secret scanning on its plan, so +gitleaks is its only secret check. + +New repositories take their settings from the organisation's code security configuration +`WireCat public`, set as the default for new public repositories; settings applied repository by +repository are not inherited. Check that it is the default and that it keeps CodeQL default setup +off: + +```sh +gh api orgs/WireCatLabs/code-security/configurations/defaults --jq '.[] | + "\(.default_for_new_repos) \(.configuration.name) \(.configuration.code_scanning_default_setup)"' +# public WireCat public disabled +``` ## Checklist ### S · Code and supply chain -- **S1 · Install script.** `install/postinstall.mjs` in both CLIs runs on every user's machine at - `npm i -g`. What it touches, downloads, executes; whether it fails open. -- **S2 · Self-update** (`src/update.ts`). What it runs, whether it checks integrity, whether a - version string reaches a shell. +- **S1 · Install scripts.** `install/postinstall.mjs` in max-cli, tg-cli and cli-memo runs on every + user's machine at `npm i -g`; the Windows installers (`install/windows.ps1` in max-cli and tg-cli, + `public/install.ps1` in cli-docs) run as a script block. What they touch, download, execute; + whether they fail open; what anchors their integrity. +- **S2 · Self-update** (`src/update/` in cli-base, `src/update.ts` in max-cli and tg-cli). What it + runs, whether it checks integrity, whether a version string reaches a shell. - **S3 · Process spawning.** Every `child_process` / `cross-spawn` call: shell use, argument injection, `$EDITOR` / `$BROWSER` handling. - **S4 · Local servers.** MCP server and `max-cli/src/server/start.ts`: bind address, auth. -- **S5 · Credentials at rest.** MAX token, mtcute session, bot tokens: file mode, location, log - redaction (`pino`), what `diagnose` prints. +- **S5 · Credentials at rest.** MAX token, mtcute session, bot tokens, OAuth tokens of drive-cli and + zoom-cli: file and folder mode, location, log redaction (cli-base `src/logging.ts`), what + `max doctor report create` and `tg doctor report create` write. - **S6 · Downloads to disk.** Attachment names: path traversal, symlinks, overwriting. -- **S7 · Network clients.** `ws`, `undici`, the tg-cli proxy: TLS verification, requests to a host - an attacker chooses. -- **S8 · Parsers of untrusted data.** Wire frames (`@msgpack/msgpack`, `lossless-json`); - `@xmldom/xmldom`, `liquidjs` (who controls templates), `yaml`, `chardet`, `iconv-lite`, +- **S7 · Network clients.** `ws` (max-cli), `undici` (tg-cli), the tg-cli proxy: TLS verification, + requests to a host an attacker chooses. +- **S8 · Parsers of untrusted data.** Wire frames in max-cli (`@msgpack/msgpack`, `lossless-json`); + in cli-core `@xmldom/xmldom`, `liquidjs` (who controls templates), `yaml`, `chardet`, `iconv-lite`, `image-size`, `fflate` (zip bombs), `@resvg/resvg-js` (SVG); model downloads for ONNX and `sherpa-onnx` — source and integrity. -- **S9 · CI and release.** Job permissions, `pull_request_target`, expressions in `run:`, cache - poisoning, workflows that run an AI agent. +- **S9 · CI, release and deploy.** Job permissions, `pull_request_target`, expressions in `run:`, + cache poisoning, workflows that run an AI agent, where deploy and publish secrets live. - **S10 · Third-party tree.** Advisories, behaviour, abandoned upstreams. Source reading only for packages S1–S9 point at. @@ -132,13 +182,14 @@ question for every item: can that text make the agent do something the owner did - **A3 · Spoofed structure.** Text that imitates the tool's own output: a fake end of a JSON block, a fake "system" line, a fake tool result, markdown that hides text. - **A4 · Write gating.** Every tool that sends, edits, deletes, forwards, reacts or marks read - honours `max agent config` permissions; the default is safe; an agent cannot change its own - permissions through MCP or a command it is allowed to run. + honours the profile's permissions (`max config set permissions …`, and `max mcp --permission + ` for one server); the default is safe; an agent cannot change its own permissions + through MCP or a command it is allowed to run. - **A5 · Exfiltration.** Can an injected message make an agent forward or quote private content to another chat, a URL (link previews, downloads), or a file path? - **A6 · Tool annotations.** `readOnlyHint` / `destructiveHint` / `openWorldHint` match what each tool really does, so MCP clients ask the user before writes. - **A7 · Live test.** Injection payloads placed in the **test** chats only, then a real agent (`claude -p` with the MCP server) asked to do an ordinary read task. Pass: it reads, reports the - payload as content, and does nothing else. Run through the `test-live` skill, with the owner's - yes. Payload set kept in `agent-payloads.md` so every run uses the same ones. + payload as content, and does nothing else. Run with `cli-testing agent + --live` ([`AGENT-TEST.md`](AGENT-TEST.md)), with the owner's yes. Payload set kept in `agent-payloads.md` so every run uses the same ones. diff --git a/docs/security/RUNBOOK.md b/docs/security/RUNBOOK.md index e73d986..f5105d8 100644 --- a/docs/security/RUNBOOK.md +++ b/docs/security/RUNBOOK.md @@ -6,10 +6,12 @@ first run was 2026-10-09; its results are private (`cli-private/security/runs/20 ## 0. Tools, pinned A version at least two weeks old on the day of the run, recorded in `SCOPE.md` by the scripts. +`cli-testing security scan` refuses any other version. ```sh uv tool install semgrep==1.178.0 uv tool install zizmor==1.30.1 +# gitleaks: the release binary, checked against its checksums file the same way as osv-scanner # osv-scanner: the release binary, checked against the release's own SHA256SUMS V=2.6.0; D=$(mktemp -d); cd "$D" gh release download v$V -R google/osv-scanner -p osv-scanner_linux_amd64 -p osv-scanner_SHA256SUMS @@ -26,7 +28,9 @@ Disable `fp-check` again after the run — its Stop hook fires on every session. ## 1. GitHub settings — apply, then read back -Per repository `/`: +New public repositories get their settings from the organisation configuration `WireCat public`; +check that it is still the default with the command in [`METHOD.md`](METHOD.md#github-settings). +For a repository whose read-back shows a setting off, per repository `/`: ```sh r=/ @@ -35,26 +39,46 @@ gh api -X PATCH repos/$r --input - <<'EOF' EOF gh api -X PUT repos/$r/vulnerability-alerts gh api -X PUT repos/$r/automated-security-fixes -gh api -X PATCH repos/$r/code-scanning/default-setup -f state=configured -f query_suite=extended +gh api -X PUT repos/$r/private-vulnerability-reporting +``` + +A repository that publishes to npm also gets the `npm` environment, limited to `main`: + +```sh gh api -X PUT repos/$r/environments/npm --input - <<'EOF' {"deployment_branch_policy":{"protected_branches":false,"custom_branch_policies":true}} EOF gh api -X POST repos/$r/environments/npm/deployment-branch-policies -f name=main -f type=branch ``` -Read back with the four commands in [`METHOD.md`](METHOD.md#github-settings). A repository whose +CodeQL default setup stays off: CodeQL runs in `release-checks.yml`, and default setup would take +over the alerts that workflow uploads. Read back with the commands in +[`METHOD.md`](METHOD.md#github-settings); default setup reads `not-configured`. A repository whose `dependabot.yml` limits npm updates with `allow` needs `target-branch: main` on that entry, or the `allow` list also filters security updates ([`LESSONS.md`](LESSONS.md)). -## 2. Scanners and Socket +## 2. Scanners, Socket and gitleaks + +Every public repository, each checkout in a folder named after its repository (the runner refuses +two checkouts with the same folder name): ```sh RUN=~/Projects/AI/cli-private/security/runs/$(date +%F) -C=(~/Projects/AI/max-cli ~/Projects/AI/tg-cli ~/Projects/AI/cli-messaging ~/Projects/AI/cli-base) -scripts/security/scan "$RUN" "${C[@]}" -scripts/security/socket "$RUN" "${C[@]}" +P=~/Projects/AI +C=($P/max-cli $P/tg-cli $P/cli-core $P/cli-base $P/cli-memo $P/drive-cli $P/zoom-cli $P/cli-docs + $P/cli-testing $P/community $P/.github) +cli-testing security scan "$RUN" "${C[@]}" --previous +cli-testing security socket "$RUN" "${C[@]}" +for c in "${C[@]}" $P/cli-private; do + (cd "$c" && gitleaks git --no-banner --redact .; echo "$(basename "$c") gitleaks exit=$?") +done ``` +A failed repository does not stop the others: `scan` names it on stderr and exits 1 after the last +one. Rerun that repository into a new run folder once the cause is fixed. gitleaks runs from each +repository's root, where it finds that repository's `.gitleaksignore`; record each exit code and +hit count in `SCOPE.md`. + ## 3. CodeQL triage Group the open alerts per repository and rule from `raw//codeql.json`. For each alert decide: diff --git a/package.json b/package.json index a65db55..0f2f406 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "README.md", "LICENSE", "scripts/security/scan", - "scripts/security/semgrep.ts", "scripts/security/socket", "scripts/agent/run-agent", "scripts/agent/run-payload", diff --git a/scripts/security-scan.test.ts b/scripts/security-scan.test.ts new file mode 100644 index 0000000..e0e7b27 --- /dev/null +++ b/scripts/security-scan.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from "node:child_process" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { describe, expect, it } from "vitest" + +const git = (cwd: string, ...args: string[]) => { + const result = spawnSync( + "git", + ["-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", ...args], + { cwd, encoding: "utf8" }, + ) + if (result.status !== 0) throw new Error(result.stderr) +} + +const fixture = () => { + const root = mkdtempSync(join(tmpdir(), "security-scan-")), + bin = join(root, "bin") + mkdirSync(bin) + const executable = (name: string, body: string) => + writeFileSync(join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o700 }) + executable("osv-scanner", 'echo "{}"') + executable("zizmor", 'echo "[]"') + executable("gh", 'case "$*" in *code-scanning/alerts*) echo "[[]]" ;; *) echo synthetic ;; esac') + executable( + "semgrep", + `case "$PWD" in */broken) exit 2 ;; esac +echo '{"results":[],"errors":[{"type":"Internal matching error","level":"warn","rule_id":"synthetic.rule","path":"ci.yml"}],"paths":{"scanned":["a.js"]}}'`, + ) + const checkout = (name: string) => { + const origin = join(root, "origin", name), + path = join(root, name) + mkdirSync(origin, { recursive: true }) + git(origin, "init", "-q", "-b", "main") + writeFileSync(join(origin, "a.js"), "export const a = 1\n") + writeFileSync(join(origin, ".semgrepignore"), "# synthetic\nfixtures/\n") + mkdirSync(join(origin, ".github")) + writeFileSync(join(origin, ".github/ci.yml"), "on: push\n") + git(origin, "add", ".") + git(origin, "commit", "-q", "-m", "fixture") + git(root, "clone", "-q", origin, path) + return path + } + return { root, bin, repositories: [checkout("broken"), checkout("good")] } +} + +describe("security scan", () => { + it("records a failed repository, scans the rest, then exits 1", () => { + const { root, bin, repositories } = fixture() + const run = join(root, "run") + const result = spawnSync("bash", [resolve("scripts/security/scan"), run, ...repositories], { + encoding: "utf8", + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, + }) + expect(result.status).toBe(1) + expect(result.stderr).toContain("failed: broken semgrep") + expect(existsSync(join(run, "raw/good/semgrep.json"))).toBe(true) + expect(existsSync(join(run, "raw/broken/semgrep.json"))).toBe(false) + const scope = readFileSync(join(run, "SCOPE.md"), "utf8") + expect(scope).toContain("- broken semgrep exit=1") + expect(scope).toContain("- broken zizmor exit=0") + expect(scope).toContain("- good semgrep exit=0") + expect(scope).toContain("- good semgrep ignores fixtures/") + expect(scope).toContain("- good semgrep gap synthetic.rule ci.yml") + expect(scope).toContain("- good codeql open_alerts=0") + }) +}) diff --git a/scripts/security-semgrep.test.ts b/scripts/security-semgrep.test.ts deleted file mode 100644 index 8d650b9..0000000 --- a/scripts/security-semgrep.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { describe, expect, it } from "vitest" -import { scanSemgrep } from "./security/semgrep.ts" - -const fixture = () => { - const directory = mkdtempSync(join(tmpdir(), "scanner-runtime-")) - const root = join(directory, "source") - mkdirSync(join(root, "src"), { recursive: true }) - writeFileSync(join(root, "src/bad.ts"), "const value = request.input\nexec(value)\n") - return { root, output: join(directory, "raw/semgrep.json"), supplement: join(directory, "supplement") } -} -const result = (errors: unknown[] = [], results: unknown[] = [], scanned = ["src/bad.ts"]) => - JSON.stringify({ results, errors, paths: { scanned } }) -const error = { type: "Syntax error", path: "src/bad.ts" } - -describe("Semgrep runtime fallback", () => { - it("retains source failures and maps a positive runtime control to its original source", () => { - const { root, output, supplement } = fixture() - let scans = 0 - const done = scanSemgrep(root, output, supplement, (command, args) => { - if (command === "pnpm") { - expect(args).toContain("esbuild@0.25.12") - expect(args).toContain("./src/bad.ts") - const runtime = join(supplement, "runtime/src") - mkdirSync(runtime, { recursive: true }) - writeFileSync(join(runtime, "bad.js"), "exec(value);\n") - writeFileSync( - join(runtime, "bad.js.map"), - JSON.stringify({ - version: 3, - sources: ["bad.ts"], - names: [], - mappings: "AACA", - sourcesContent: ["const value = request.input\nexec(value)\n"], - }), - ) - return { code: 0, stdout: "" } - } - return { - code: 0, - stdout: - ++scans === 1 - ? result([error]) - : result( - [], - [ - { - check_id: "synthetic.unsafe-exec", - path: "src/bad.js", - start: { line: 1, col: 1 }, - end: { line: 1, col: 5 }, - }, - ], - ["src/bad.js"], - ), - } - }) - expect(done.errors).toEqual([]) - expect(done.results[0]).toMatchObject({ path: "src/bad.ts", start: { line: 2, col: 1 } }) - expect(JSON.parse(readFileSync(join(supplement, "source.json"), "utf8")).errors).toEqual([error]) - expect(done.runtimeCoverage).toMatchObject({ sources: ["src/bad.ts"], typeSyntaxAnalyzed: false }) - }) - - it.each(["compile", "parse", "missing"])("does not write a successful result after a %s failure", (failure) => { - const { root, output, supplement } = fixture() - let scans = 0 - expect(() => - scanSemgrep(root, output, supplement, (command) => { - if (command === "pnpm") return { code: failure === "compile" ? 1 : 0, stdout: "" } - return { code: 0, stdout: ++scans === 1 ? result([error]) : result(failure === "parse" ? [error] : [], [], []) } - }), - ).toThrow() - expect(existsSync(output)).toBe(false) - }) - - it("rejects a scanner error path outside the export before starting a compiler", () => { - const { root, output, supplement } = fixture() - expect(() => - scanSemgrep(root, output, supplement, (command) => { - expect(command).toBe("semgrep") - return { code: 0, stdout: result([{ type: "Syntax error", path: "../outside.ts" }]) } - }), - ).toThrow("outside the source export") - }) -}) diff --git a/scripts/security/scan b/scripts/security/scan index 01e7bb6..57d8897 100755 --- a/scripts/security/scan +++ b/scripts/security/scan @@ -47,15 +47,17 @@ done echo for checkout in "$@"; do s="$(slug "$checkout")" - a="$(gh api "repos/$s" --jq '.security_and_analysis | "secret_scanning=\(.secret_scanning.status) push_protection=\(.secret_scanning_push_protection.status) security_updates=\(.dependabot_security_updates.status)"')" - c="$(gh api "repos/$s/code-scanning/default-setup" --jq .state)" + a="$(gh api "repos/$s" --jq '.security_and_analysis | "secret_scanning=\(.secret_scanning.status) push_protection=\(.secret_scanning_push_protection.status) security_updates=\(.dependabot_security_updates.status)"')" || a="unreadable" + c="$(gh api "repos/$s/code-scanning/default-setup" --jq .state)" || c="unreadable" echo "- $s: $a codeql=$c" done } >> "$scope" -# Fails the run on any exit code the tool's docs do not list as a result, so a crash never reads as -# "no findings": osv-scanner 0/1, semgrep 0 (no --error), zizmor 0 or 11-14. +# A tool exit code its docs do not list as a result marks that repository failed, so a crash never +# reads as "no findings": osv-scanner 0/1, semgrep 0 (no --error), zizmor 0 or 11-14. The other +# repositories are still scanned, and the run exits 1 at the end. status=() +failed=() run_tool() { local tool="$1" ok="$2" file="$3"; shift 3 local code=0 @@ -63,10 +65,18 @@ run_tool() { status+=("$name $tool exit=$code") if [[ " $ok " != *" $code "* ]]; then echo "$name: $tool failed with exit $code" >&2 - exit 1 + failed+=("$name $tool") + return 1 fi } +# A checkout runs the source; the published package has only the compiled file, because Node does +# not strip types under node_modules. +semgrep_runner=(node "$here/../../dist/semgrep.js") +if [ -f "$here/../../src/semgrep.ts" ]; then + semgrep_runner=(node --experimental-strip-types "$here/../../src/semgrep.ts") +fi + i=0 for checkout in "$@"; do name="${names[$i]}"; i=$((i + 1)) @@ -74,15 +84,27 @@ for checkout in "$@"; do mkdir -p "$out" cd "$work/$name" if [ -f pnpm-lock.yaml ]; then - run_tool osv-scanner "0 1" "$out/osv.json" osv-scanner scan source -r --format json . + run_tool osv-scanner "0 1" "$out/osv.json" osv-scanner scan source -r --format json . || true + fi + if [ -f .semgrepignore ]; then + status+=("$name semgrep ignores $(sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' .semgrepignore | paste -sd ' ' -)") + fi + if run_tool semgrep "0" /dev/null "${semgrep_runner[@]}" "$work/$name" "$out/semgrep.json" "$run/supplement/$name"; then + while IFS= read -r gap; do + status+=("$name semgrep gap $gap") + done < <(jq -r '.coverageGaps // [] | .[] | "\(.rule) \(.path)"' "$out/semgrep.json") fi - run_tool semgrep "0" /dev/null node --experimental-strip-types "$here/semgrep.ts" \ - "$work/$name" "$out/semgrep.json" "$run/supplement/$name" if [ -d .github ]; then - run_tool zizmor "0 11 12 13 14" "$out/zizmor.json" zizmor --offline --format json .github + run_tool zizmor "0 11 12 13 14" "$out/zizmor.json" zizmor --offline --format json .github || true + fi + if gh api --paginate --slurp "repos/$(slug "$checkout")/code-scanning/alerts?state=open&per_page=100" > "$out/codeql.json"; then + alerts="$(jq 'flatten | length' "$out/codeql.json")" || alerts="unreadable" + status+=("$name codeql open_alerts=$alerts") + else + echo "$name: reading CodeQL alerts failed" >&2 + rm -f "$out/codeql.json" + failed+=("$name codeql") fi - gh api --paginate --slurp "repos/$(slug "$checkout")/code-scanning/alerts?state=open&per_page=100" > "$out/codeql.json" - status+=("$name codeql open_alerts=$(jq 'flatten | length' "$out/codeql.json")") echo "$name done" >&2 done @@ -96,4 +118,8 @@ find "$run/raw" -name '*.json' -exec sed -i "s|$work/[^/\"]*/||g" {} + for s in "${status[@]}"; do echo "- $s"; done } >> "$scope" +if [ "${#failed[@]}" -gt 0 ]; then + printf 'failed: %s\n' "${failed[@]}" >&2 + exit 1 +fi echo "$run" >&2 diff --git a/src/semgrep.test.ts b/src/semgrep.test.ts new file mode 100644 index 0000000..a0f45ec --- /dev/null +++ b/src/semgrep.test.ts @@ -0,0 +1,170 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { scanSemgrep } from "./semgrep.js" + +const fixture = () => { + const directory = mkdtempSync(join(tmpdir(), "scanner-runtime-")) + const root = join(directory, "source") + mkdirSync(join(root, "src"), { recursive: true }) + writeFileSync(join(root, "src/bad.ts"), "const value = request.input\nexec(value)\n") + return { root, output: join(directory, "raw/semgrep.json"), supplement: join(directory, "supplement") } +} +const result = (errors: unknown[] = [], results: unknown[] = [], scanned = ["src/bad.ts"]) => + JSON.stringify({ results, errors, paths: { scanned } }) +const error = { type: "Syntax error", path: "src/bad.ts" } + +describe("Semgrep runtime fallback", () => { + it("retains source failures and maps a positive runtime control to its original source", () => { + const { root, output, supplement } = fixture() + let scans = 0 + const done = scanSemgrep(root, output, supplement, (command, args) => { + if (command === "pnpm") { + expect(args).toContain("esbuild@0.25.12") + expect(args).toContain("./src/bad.ts") + const runtime = join(supplement, "runtime/src") + mkdirSync(runtime, { recursive: true }) + writeFileSync(join(runtime, "bad.js"), "exec(value);\n") + writeFileSync( + join(runtime, "bad.js.map"), + JSON.stringify({ + version: 3, + sources: ["bad.ts"], + names: [], + mappings: "AACA", + sourcesContent: ["const value = request.input\nexec(value)\n"], + }), + ) + return { code: 0, stdout: "" } + } + return { + code: 0, + stdout: + ++scans === 1 + ? result([error]) + : result( + [], + [ + { + check_id: "synthetic.unsafe-exec", + path: "src/bad.js", + start: { line: 1, col: 1 }, + end: { line: 1, col: 5 }, + }, + ], + ["src/bad.js"], + ), + } + }) + expect(done.errors).toEqual([]) + expect(done.results[0]).toMatchObject({ path: "src/bad.ts", start: { line: 2, col: 1 } }) + expect(JSON.parse(readFileSync(join(supplement, "source.json"), "utf8")).errors).toEqual([error]) + expect(done.runtimeCoverage).toMatchObject({ sources: ["src/bad.ts"], typeSyntaxAnalyzed: false }) + }) + + it.each(["compile", "parse", "missing"])("does not write a successful result after a %s failure", (failure) => { + const { root, output, supplement } = fixture() + let scans = 0 + expect(() => + scanSemgrep(root, output, supplement, (command) => { + if (command === "pnpm") return { code: failure === "compile" ? 1 : 0, stdout: "" } + return { code: 0, stdout: ++scans === 1 ? result([error]) : result(failure === "parse" ? [error] : [], [], []) } + }), + ).toThrow() + expect(existsSync(output)).toBe(false) + }) + + it("rejects a scanner error path outside the export before starting a compiler", () => { + const { root, output, supplement } = fixture() + expect(() => + scanSemgrep(root, output, supplement, (command) => { + expect(command).toBe("semgrep") + return { code: 0, stdout: result([{ type: "Syntax error", path: "../outside.ts" }]) } + }), + ).toThrow("outside the source export") + }) + + it("compiles a JavaScript module the parser rejects, keeping top-level await, and maps it back", () => { + const { root, output, supplement } = fixture() + writeFileSync(join(root, "src/tool.mjs"), "const data = await load()\neval(data)\n") + const syntax = { type: "Syntax error", path: "src/tool.mjs" } + let scans = 0 + const done = scanSemgrep(root, output, supplement, (command, args) => { + if (command === "pnpm") { + expect(args).toEqual(expect.arrayContaining(["./src/tool.mjs", "--supported:top-level-await=true"])) + const runtime = join(supplement, "runtime/src") + mkdirSync(runtime, { recursive: true }) + writeFileSync(join(runtime, "tool.js"), "const data = await load();\neval(data);\n") + writeFileSync( + join(runtime, "tool.js.map"), + JSON.stringify({ version: 3, sources: ["tool.mjs"], names: [], mappings: "AAAA;AACA", sourcesContent: [""] }), + ) + return { code: 0, stdout: "" } + } + if (++scans === 1) return { code: 0, stdout: result([syntax]) } + expect(args.at(-1)).toBe("./src/tool.js") + const match = { + check_id: "synthetic.eval", + path: "src/tool.js", + start: { line: 2, col: 1 }, + end: { line: 2, col: 5 }, + } + return { code: 0, stdout: result([], [match], ["src/tool.js"]) } + }) + expect(done.results[0]).toMatchObject({ path: "src/tool.mjs", start: { line: 2, col: 1 } }) + expect(done.runtimeCoverage).toMatchObject({ sources: ["src/tool.mjs"] }) + }) + + it("refuses two failed sources that would compile to the same runtime file", () => { + const { root, output, supplement } = fixture() + expect(() => + scanSemgrep(root, output, supplement, (command) => { + expect(command).toBe("semgrep") + return { + code: 0, + stdout: result([ + { type: "Syntax error", path: "src/bad.ts" }, + { type: "Syntax error", path: "src/bad.mjs" }, + ]), + } + }), + ).toThrow("same runtime file") + expect(existsSync(output)).toBe(false) + }) + + it("records one rule failing on one file as a coverage gap and keeps the scan", () => { + const { root, output, supplement } = fixture() + const warning = { + type: "Internal matching error", + level: "warn", + rule_id: "synthetic.rule", + path: ".github/workflows/ci.yml", + } + const done = scanSemgrep(root, output, supplement, (command) => { + expect(command).toBe("semgrep") + return { code: 0, stdout: result([warning]) } + }) + expect(done.coverageGaps).toEqual([ + { type: "Internal matching error", rule: "synthetic.rule", path: ".github/workflows/ci.yml" }, + ]) + expect(JSON.parse(readFileSync(output, "utf8")).errors).toEqual([warning]) + }) + + it.each([ + ["a timeout", { type: "Timeout", path: "src/bad.ts" }], + ["a matching error at error level", { type: "Internal matching error", level: "error", rule_id: "r", path: "a" }], + ["a syntax error outside JavaScript", { type: "Syntax error", path: "config.yml" }], + ])("fails closed on %s", (_, failure) => { + const { root, output, supplement } = fixture() + expect(() => scanSemgrep(root, output, supplement, () => ({ code: 0, stdout: result([failure]) }))).toThrow( + "unrecoverable", + ) + expect(existsSync(output)).toBe(false) + }) + + it("fails closed when the scanner itself exits with an error", () => { + const { root, output, supplement } = fixture() + expect(() => scanSemgrep(root, output, supplement, () => ({ code: 2, stdout: "" }))).toThrow("exit 2") + }) +}) diff --git a/scripts/security/semgrep.ts b/src/semgrep.ts similarity index 78% rename from scripts/security/semgrep.ts rename to src/semgrep.ts index 4740a57..74037f2 100644 --- a/scripts/security/semgrep.ts +++ b/src/semgrep.ts @@ -1,12 +1,14 @@ import { spawnSync } from "node:child_process" -import { mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs" import { SourceMap } from "node:module" -import { dirname, extname, isAbsolute, relative, resolve } from "node:path" +import { dirname, isAbsolute, relative, resolve } from "node:path" import { fileURLToPath } from "node:url" const PACKS = ["p/javascript", "p/typescript", "p/nodejs", "p/secrets", "p/github-actions"] const ESBUILD = "0.25.12" -type Failure = { type: string | unknown[]; path?: string } +const SOURCES = /\.(?:[cm]?[jt]s|[jt]sx)$/ +type Failure = { type: string | unknown[]; path?: string; level?: string; rule_id?: string } +type Gap = { type: string; rule: string; path: string } type Match = { check_id: string path: string @@ -26,6 +28,12 @@ const run: Run = (command, args, cwd) => { return { code: result.status ?? 2, stdout: result.stdout ?? "" } } const partial = (error: Failure) => Array.isArray(error.type) && error.type[0] === "PartialParsing" +// One rule failing on one file is recorded as a gap; any other engine error still fails the scan. +const gap = (error: Failure) => + error.type === "Internal matching error" && + error.level === "warn" && + typeof error.rule_id === "string" && + typeof error.path === "string" const within = (path: string, root: string) => { const candidate = relative(root, resolve(root, path)) if (!candidate || candidate.startsWith("..") || isAbsolute(candidate)) @@ -56,19 +64,24 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in return value } const source = scan(root) - const failures = source.errors.filter((error) => !partial(error)) + const gaps: Gap[] = source.errors.filter(gap).map((error) => ({ + type: error.type as string, + rule: error.rule_id as string, + path: within(error.path as string, root), + })) + const failures = source.errors.filter((error) => !partial(error) && !gap(error)) mkdirSync(supplement, { recursive: true, mode: 0o700 }) writeFileSync(resolve(supplement, "source.json"), `${JSON.stringify(source)}\n`, { mode: 0o600 }) if (failures.length) { - if ( - failures.some( - (error) => error.type !== "Syntax error" || !error.path || ![".ts", ".tsx"].includes(extname(error.path)), - ) - ) + if (failures.some((error) => error.type !== "Syntax error" || !error.path || !SOURCES.test(error.path))) throw new Error("Semgrep reported an unrecoverable error") const files = [...new Set(failures.map((error) => within(error.path as string, root)))] + const expected = files.map((file) => file.replace(SOURCES, ".js")) + if (new Set(expected).size !== expected.length) + throw new Error("two failed sources compile to the same runtime file") const compiled = resolve(supplement, "runtime") // ES2019 lowers the syntax rejected by the pinned parser; imports are never bundled or executed. + // Top-level await cannot be lowered, and the parser accepts it. const build = invoke( "pnpm", [ @@ -79,6 +92,7 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in "--outbase=.", "--platform=node", "--target=es2019", + "--supported:top-level-await=true", "--format=esm", "--charset=ascii", "--sourcemap=external", @@ -88,7 +102,6 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in root, ) if (build.code !== 0) throw new Error(`runtime coverage compilation failed with exit ${build.code}`) - const expected = files.map((file) => file.replace(/\.(?:tsx|ts)$/, ".js")) const fallback = scan( compiled, expected.map((file) => `./${file}`), @@ -117,7 +130,7 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in } source.results.push({ ...match, path: sourcePath, start: mapped(match.start), end: mapped(match.end) }) } - source.errors = source.errors.filter(partial) + source.errors = source.errors.filter((error) => partial(error) || gap(error)) source.runtimeCoverage = { compiler: `esbuild ${ESBUILD}`, target: "es2019", @@ -127,6 +140,7 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in runtimeResults: relative(dirname(output), resolve(supplement, "runtime.json")), } } + if (gaps.length) source.coverageGaps = gaps source.results = [ ...new Map( source.results.map((match) => [ @@ -140,7 +154,11 @@ export const scanSemgrep = (root: string, output: string, supplement: string, in return source } -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { +if ( + process.argv[1] && + existsSync(process.argv[1]) && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) { try { scanSemgrep( resolve(process.argv[2] as string),