diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2ac6aee..2138fed 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,9 +1,4 @@ -# Monthly update PRs into develop: one batch a month, merged as one PR so develop deploys once. -# The cooldown holds a release back for 7 days so a broken or hijacked version is caught upstream -# first; security updates ignore both the schedule and the cooldown. -# -# Every ignore rule below is a deliberate hold; its reason is the -# comment next to it. +# The 7-day cooldown lets a broken or hijacked release be caught upstream first. version: 2 updates: - package-ecosystem: "nuget" @@ -21,7 +16,7 @@ updates: patterns: ["*"] exclude-patterns: ["QuestPDF"] update-types: [minor, patch] - # Always alone: a QuestPDF bump must pass .github/scripts/questpdf-cpu-check.sh before it is merged + # Alone: a QuestPDF bump must pass .github/scripts/questpdf-cpu-check.sh before merging questpdf: patterns: ["QuestPDF"] ignore: @@ -79,8 +74,7 @@ updates: - dependency-name: "@types/node" versions: [">=23"] - # One entry for all three Dockerfiles: the SDK and runtime references are shared between - # Backend and the QuestPDF harness and must move in one PR (see "SDK image" in the register) + # One entry so the SDK/runtime references shared across Dockerfiles move in one PR - package-ecosystem: "docker" directories: - "/src/backend" @@ -95,8 +89,7 @@ updates: commit-message: prefix: "chore(deps)" groups: - # Dependabot names a Docker dependency by its image path without the registry: - # dotnet/sdk and dotnet/aspnet, not mcr.microsoft.com/dotnet/... + # Dependabot names Docker images without the registry (dotnet/sdk, not mcr.microsoft.com/...) dotnet-images: patterns: ["dotnet/*"] other-images: diff --git a/.github/scripts/check-deps.sh b/.github/scripts/check-deps.sh index 6c2bb5c..36b20f7 100644 --- a/.github/scripts/check-deps.sh +++ b/.github/scripts/check-deps.sh @@ -1,32 +1,6 @@ #!/usr/bin/env bash -# Supply-chain gate for pkgdiff: NuGet, npm and (optionally) the built images. -# -# bash .github/scripts/check-deps.sh run from any directory -# -# TRIVY=1 also scan ${BACKEND_IMAGE:-ghcr.io/pkgdiff-dev/pkgdiff-backend:local} and -# ${FRONTEND_IMAGE:-ghcr.io/pkgdiff-dev/pkgdiff-frontend:local} with the -# pinned Trivy image (the defaults are what docker-compose.build.yml builds) -# CHECK_DEPS_DOCKER=auto auto (default): use dotnet / npm from PATH, and the pinned SDK / Node -# image for a tool that is missing; 1: always containers; 0: never -# PKGDIFF_ROOT= check another checkout (default: the checkout this script is in) -# -# Checks (every check runs, also after a failure): -# sdk-image-pin the SDK image is pinned by digest and is the same reference in -# src/backend/Dockerfile, src/backend/Directory.Build.props and the QuestPDF harness -# nuget-locked-restore dotnet restore --locked-mode with the audit gate (lock drift, NU1901-NU1904) -# nuget-vulnerable dotnet list package --vulnerable --include-transitive, read as JSON -# npm-audit-prod npm audit --omit=dev (production dependencies must be clean) -# npm-audit-high .github/scripts/npm-audit-gate.mjs with src/frontend/audit-allowlist.json -# trivy-backend TRIVY=1 only: fixable HIGH/CRITICAL in the backend image -# trivy-frontend TRIVY=1 only: fixable HIGH/CRITICAL in the frontend image -# -# Output: one "PASS " or "FAIL : " line per check (evidence is indented -# above it), then "check-deps: N passed, M failed". A check that could not be carried out is a -# FAIL whose detail starts with "could not check"; it is never reported as a pass. -# Exit status: 0 when every check passed, otherwise 1. -# -# The checkout is never modified: the manifests are copied to a temporary directory and the -# tools run on that copy, either on this machine or inside a container. +# Supply-chain gate: NuGet and npm audits, plus image scans with TRIVY=1. Exit 0 when all pass. +# Env: TRIVY=1, CHECK_DEPS_DOCKER=auto|1|0, PKGDIFF_ROOT=, BACKEND_IMAGE, FRONTEND_IMAGE. set -uo pipefail export DOTNET_CLI_UI_LANGUAGE=en DOTNET_NOLOGO=1 DOTNET_CLI_TELEMETRY_OPTOUT=1 @@ -51,8 +25,7 @@ case "$MODE" in ;; esac -# Git Bash rewrites arguments that look like POSIX paths (/work, /var/run/docker.sock) before -# docker.exe sees them; turn that off and use the socket form Docker Desktop accepts. +# Git Bash rewrites POSIX-looking arguments before docker.exe sees them; turn that off. DOCKER_SOCKET='/var/run/docker.sock' case "$(uname -s 2>/dev/null)" in MINGW* | MSYS* | CYGWIN*) @@ -121,7 +94,6 @@ one_line() { tr '\r\n' ' ' | sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' } -# Prints the image of the "AS build" stage of a Dockerfile. build_stage_image() { [ -f "$1" ] || return 1 tr -d '\r' <"$1" | @@ -129,13 +101,11 @@ build_stage_image() { head -n 1 } -# A status that decides something never comes from `writer | grep -q`: under pipefail a writer -# killed by SIGPIPE makes a match read as "no match". These tests read a here-string or a file. +# No `writer | grep -q` for decisions: under pipefail a SIGPIPE'd writer turns a match into a miss. is_pinned() { grep -Eq '^[A-Za-z0-9][A-Za-z0-9./_-]*:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$' <<<"$1" } -# Decides whether a tool set runs in a container: 0 = container, 1 = this machine. use_container() { local tool case "$MODE" in @@ -148,9 +118,7 @@ use_container() { return 1 } -# run_in_container [docker create options] -# Copies to /work in a fresh container, runs /work/run.sh there and copies /work/out back. -# No bind mount is used, so nothing depends on how this machine shares its file system with Docker. +# Copies files in and out instead of bind-mounting, so host file sharing with Docker doesn't matter. run_in_container() { local image="$1" dir="$2" cid rc shift 2 @@ -169,8 +137,6 @@ run_in_container() { return "$rc" } -# ---------------------------------------------------------------- sdk-image-pin - check_sdk_image_pin() { local name='sdk-image-pin' local dockerfile="$ROOT/src/backend/Dockerfile" @@ -205,8 +171,6 @@ check_sdk_image_pin() { pass "$name" } -# ---------------------------------------------------------------- NuGet - prepare_nuget() { local dir="$1" rel mkdir -p "$dir" || return 1 @@ -240,7 +204,6 @@ echo "$?" >out/vulnerable.rc EOF } -# Unique "error XX0000: message" lines of a restore log, without the trailing "[project path]". restore_errors() { tr -d '\r' <"$1" | sed -n 's/.*\(error [A-Z][A-Z]*[0-9][0-9]*: .*\)$/\1/p' | sed 's/ \[[^][]*\]$//' | sort -u } @@ -286,11 +249,9 @@ check_nuget() { return fi - # nuget-locked-restore rc="$(read_rc "$out/locked-restore.rc")" if [ "$rc" = "0" ] && grep -Eq 'NU190[05]' "$out/locked-restore.log"; then - # With packages in the local cache, a restore still succeeds when the advisory data cannot be - # downloaded; it only warns (NU1900 / NU1905). The audit did not run, so this is not a pass. + # A cached restore succeeds with only a warning when advisory data can't load: no audit ran. tr -d '\r' <"$out/locked-restore.log" | grep -oE 'warning NU190[05]: [^[]*' | sort -u | evidence unchecked nuget-locked-restore "the restore succeeded, but the advisory data could not be loaded (NU1900/NU1905), so the audit did not run" elif [ "$rc" = "0" ]; then @@ -315,7 +276,6 @@ check_nuget() { fi fi - # nuget-vulnerable rc="$(read_rc "$out/plain-restore.rc")" if [ "$rc" != "0" ]; then restore_errors "$out/plain-restore.log" 2>/dev/null | head -n 10 | evidence @@ -343,7 +303,6 @@ check_nuget() { return fi if grep -q '"vulnerabilities"' "$out/vulnerable.json"; then - # The report lists only affected packages, so every id / resolvedVersion pair in it is a finding. findings="$(grep -oE '"(id|resolvedVersion|severity|advisoryurl)": *"[^"]*"' "$out/vulnerable.json" | awk -F'"' ' $2 == "id" { id = $4 } @@ -358,8 +317,6 @@ check_nuget() { pass nuget-vulnerable } -# ---------------------------------------------------------------- npm - prepare_npm() { local dir="$1" file mkdir -p "$dir/Frontend" "$dir/scripts" || return 1 @@ -437,7 +394,7 @@ check_npm() { return fi - # npm-audit-prod: npm audit --omit=dev exits non-zero on findings and on errors; the JSON tells them apart. + # npm audit exits non-zero on findings and on errors; only the JSON tells them apart. rc="$(read_rc "$out/audit-prod.rc")" if ! grep -q '"auditReportVersion"' "$out/audit-prod.json" 2>/dev/null; then cat "$out/audit-prod.json" "$out/audit-prod.err" 2>/dev/null | tail -n 10 | evidence @@ -451,13 +408,9 @@ check_npm() { fail npm-audit-prod "production dependencies have advisories: $(printf '%s' "$packages" | one_line)" fi - # npm-audit-high: replaces a bare `npm audit --audit-level=high`, which cannot tell a reviewed, - # time-boxed exception from a new advisory. The gate keeps its own 0 / 1 / 2 exit status visible. rc="$(read_rc "$out/gate.rc")" case "$rc" in 0) - # Only the entries of the "Accepted exceptions" block are reviewed exceptions. The "Not gated" - # blocks list advisories below high that nobody reviewed; they keep their own label. tr -d '\r' <"$out/gate.log" | sed -n \ -e '1p' \ -e '/^Accepted exceptions/,/^$/s/^ - \(GHSA-.*\)$/accepted exception: \1/p' \ @@ -481,8 +434,6 @@ check_npm() { esac } -# ---------------------------------------------------------------- Trivy - check_trivy() { local name="$1" image="$2" log="$WORK/$1.log" rc detail @@ -513,24 +464,16 @@ check_trivy() { unchecked "$name" "trivy exit 0, but it detected no OS packages in $image, so nothing was scanned" fi elif [ "$rc" -eq 1 ] && grep -Eq '^Total: [1-9][0-9]* \(' "$log"; then - # Exit 1 together with a report is the only result that means "findings". - # grep reads the file itself: behind a pipe, `grep -q` stops at the first match, the writer - # dies of SIGPIPE on a large report, and pipefail turns the match into "false". The pattern - # has no end anchor, so a trailing CR does not matter. - # The whole report: whoever fixes the image needs the package and the fixed version. + # Only exit 1 together with a report means findings. grep -v "$(printf '\tINFO\t')" "$log" | evidence fail "$name" "fixable HIGH/CRITICAL findings in $image: $(tr -d '\r' <"$log" | grep -E '^Total: ' | one_line)" else - # No report: the scanner never started (docker run exits 125, 126 or 127 when the image cannot - # be pulled or run) or it stopped before the scan finished. Nothing is known about the image. tail -n 10 "$log" | evidence detail="$(grep -E 'FATAL|[Ee]rror' "$log" | head -n 1 | one_line)" unchecked "$name" "trivy did not complete (exit $rc)${detail:+: $detail}" fi } -# ---------------------------------------------------------------- run - echo "check-deps: root $ROOT" check_sdk_image_pin diff --git a/.github/scripts/questpdf-cpu-check.sh b/.github/scripts/questpdf-cpu-check.sh index f1f87b5..620318b 100644 --- a/.github/scripts/questpdf-cpu-check.sh +++ b/.github/scripts/questpdf-cpu-check.sh @@ -1,15 +1,5 @@ #!/usr/bin/env bash -# QuestPDF CPU gate: renders the PDF export document inside a Linux container and reports -# whether the QuestPDF native library works on a CPU without AVX2 (the production CPU is -# an Ivy Bridge Xeon: AVX, but no AVX2, FMA or BMI2). -# -# --native render on the real CPU of this machine. Run it on the production -# server before the backend is recreated with a new QuestPDF version. -# --emulate render under QEMU with the IvyBridge CPU model, after a self-test -# proves that the emulator really traps AVX2, FMA and BMI2. -# --version QuestPDF version to test (default: src/backend/Directory.Packages.props) -# -# Everything runs in containers; nothing is executed on the host itself. +# Checks in containers that QuestPDF renders on a CPU without AVX2, FMA or BMI2 (production). # Exit status: 0 = PASS, 1 = FAIL, 2 = usage error. set -euo pipefail @@ -67,8 +57,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" CONTEXT="$ROOT/.github/scripts/questpdf-cpu-check" if [ -z "$VERSION" ]; then - # sed stops at the first match itself. With `| head -n 1`, a second matching line could kill - # sed with SIGPIPE, and under `set -e -o pipefail` that ends the script without a message. + # sed quits itself: `| head -n 1` could SIGPIPE sed and silently abort under pipefail. VERSION="$(sed -n '/ typeof(QuestPDF.Settings).GetProperty(name, BindingFlags.Public | BindingFlags.Static)) .FirstOrDefault(property => property is { CanWrite: true }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b798865..a593e3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,4 @@ -# Runs on every pull request into develop or master, and as the gate of the publish workflow. -# No toolchain is installed on the runner: the tests run inside the SDK and Node images pinned in -# src/backend/Dockerfile and src/frontend/Dockerfile, so CI uses exactly what the published images use. +# PR gate and publish gate; runs inside the SDK and Node images the Dockerfiles pin. name: CI on: @@ -25,8 +23,7 @@ jobs: with: persist-credentials: false - # The database tests run against the PostgreSQL image docker-compose.yml pins; without a server - # they would be skipped. The password belongs to this throwaway container only. + # Without a database server the database tests would be skipped - name: Run the tests in the pinned SDK image, against the pinned PostgreSQL run: | sdk="$(sed -n 's/^FROM \(mcr\.microsoft\.com\/dotnet\/sdk:[^ ]*\) AS build.*/\1/p' src/backend/Dockerfile)" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e95a2ed..e7917b7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,9 +1,4 @@ -# Runs deploy/remote.sh on the server over SSH. -# - Automatically: publish.yml calls it after the images of a push to develop are published, -# while the repository variable PKGDIFF_AUTO_DEPLOY is "true". -# - By hand: Actions > Deploy > Run workflow (deploy a tag, roll back, or show the status). -# It does nothing until these repository secrets exist (docs/ops/deploy-ghcr.md): -# DEPLOY_HOST, DEPLOY_USER, DEPLOY_SSH_KEY, DEPLOY_KNOWN_HOSTS; optional DEPLOY_PORT, DEPLOY_PATH +# Runs deploy/remote.sh on the server over SSH; called by publish.yml or run by hand. name: Deploy on: @@ -39,7 +34,6 @@ jobs: environment: production runs-on: ubuntu-24.04 timeout-minutes: 20 - # One deploy at a time, in the order they were started; a running one is never cancelled concurrency: group: deploy-production cancel-in-progress: false @@ -87,10 +81,7 @@ jobs: request="$ACTION" if [ "$ACTION" = deploy ]; then request="deploy $TAG $REVISION"; fi - # Every word of the request was checked above. A key installed with remote.sh as its forced - # command ignores the script path given here and reads only the request. - # The log of this job may be public: it shows the "deploy:" lines of the server only, the full - # output (paths, container state) stays in deploy.log on the runner and on the server + # The job log may be public: show only the server's "deploy:" lines status=0 ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes \ -o ServerAliveInterval=30 -o ServerAliveCountMax=10 \ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6daf049..a24f128 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,12 +1,4 @@ -# Builds the backend and frontend images once and pushes them to GHCR, but only after the CI gate -# and the image scan pass. The server never builds: it pulls a tag with deploy/deploy.sh. -# -# A push to develop is also deployed, by deploy.yml, while the repository variable -# PKGDIFF_AUTO_DEPLOY is "true" (Settings > Secrets and variables > Actions > Variables). -# -# Tags: sha-<7 hex> on every run (never moves), the branch name (develop, master) on a branch push, -# and .. plus . on a v* tag. ":buildcache" holds build layers -# only; it is not a runnable image. +# Builds, scans and pushes the images to GHCR after the CI gate; the server only pulls them. name: Publish images on: @@ -74,11 +66,9 @@ jobs: run: | docker buildx create --name pkgdiff --driver docker-container --use created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - # Every build reruns the OS upgrade layer: a fix published during the day (libx11 on 2026-10-18) - # is in the next image, not only in tomorrow's. The layers after it are small. + # Rerun the OS upgrade layer on every build so same-day security fixes are included upgrade_key="$GITHUB_RUN_ID.$GITHUB_RUN_ATTEMPT" - # $1 image name, $2 build context, $3 build argument that reruns the OS upgrade layer build() { docker buildx build "$2" \ --tag "$IMAGE_PREFIX-$1:$PRIMARY" \ @@ -96,8 +86,6 @@ jobs: build backend src/backend APT_UPGRADE_DATE build frontend src/frontend APK_UPGRADE_DATE - # The frontend image has two roles, nginx and the render server; both have to start. The - # render server runs as it does in compose: unprivileged, on a read-only file system. - name: Start the frontend image in both roles run: | image="$IMAGE_PREFIX-frontend:$PRIMARY" @@ -115,7 +103,6 @@ jobs: docker exec pkgdiff-ssr-check wget -q -O - http://127.0.0.1:4000/terms | grep -q 'ng-server-context="ssr"' docker rm -f pkgdiff-ssr-check - # The package checks ran in the gate; this run adds the scan of the two images just built - name: Scan the images run: | CHECK_DEPS_DOCKER=1 TRIVY=1 \ @@ -146,8 +133,7 @@ jobs: echo "Deploy on the server: \`bash deploy/deploy.sh $PRIMARY\`" } >> "$GITHUB_STEP_SUMMARY" - # develop is the branch the server runs. The switch is a repository variable, so a release that - # needs a manual step on the server first can be held back without touching this file. + # A repository variable, so a release can be held back without editing this file deploy: name: Deploy needs: publish diff --git a/deploy/backup.sh b/deploy/backup.sh index 9076742..21eba4d 100644 --- a/deploy/backup.sh +++ b/deploy/backup.sh @@ -5,21 +5,8 @@ # bash deploy/backup.sh --check restore the newest nightly backup into a scratch database and report # bash deploy/backup.sh --check the same for one dump (a dump deploy/deploy.sh took works too) # bash deploy/backup.sh --list show the backups that exist -# -# A backup is a plain pg_dump, gzipped: the format deploy/deploy.sh writes before a deploy, so one -# restore procedure (docs/ops/operations.md) fits both. Nightly backups live in their own folder, -# because the deploy script keeps a fixed number of its own dumps and must not count these. -# -# A dump that cannot be taken or is cut off is deleted and the script exits 1; older backups are -# only deleted after a new one was verified. --check never touches the live database: it creates -# the scratch database, loads the dump, counts what arrived and drops the scratch database again. -# -# Environment: -# PKGDIFF_BACKUP_DIR where backups go (default ~/pkgdiff-backups; nightly ones in nightly/) -# PKGDIFF_NIGHTLY_KEEP_DAYS days a nightly backup is kept (default 14) -# PKGDIFF_DB_CONTAINER the database container (default pkgdiff-db) set -euo pipefail -# A dump holds password hashes and encrypted settings: nobody but the owner reads it, from its first byte on +# Dumps hold password hashes and encrypted settings: owner-only from the first byte umask 077 # cron runs with a short PATH @@ -71,8 +58,7 @@ nightly_files() { newest_nightly() { nightly_files | sort | tail -n 1; } -# The names carry the time of the dump, so a name is compared with the name a dump taken exactly -# KEEP_DAYS ago would have. The newest backup is never deleted, whatever the clock says. +# Names sort by dump time, so they compare against a cutoff name; the newest is always kept. prune() { local cutoff newest old cutoff="pkgdiffdb-$(date -u -d "-$KEEP_DAYS days" +%Y%m%dT%H%M%SZ).sql.gz" @@ -103,7 +89,6 @@ backup() { drop_scratch() { sql -d postgres -c "DROP DATABASE IF EXISTS $SCRATCH" >/dev/null 2>&1 || true; } -# Tables in the public schema, applied migrations, users: enough to tell a real restore from an empty one figures() { sql -d "$1" -c "SELECT (SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE') || ' tables, ' || (SELECT count(*) FROM \"__EFMigrationsHistory\") || ' migrations, ' || (SELECT count(*) FROM \"AspNetUsers\") || ' users, ' || (SELECT count(*) FROM \"CachedPackages\") || ' cached packages'" } diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 0d0325b..1d9e55f 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -6,24 +6,6 @@ # bash deploy/deploy.sh --status show the configured tag and what is running # # is one the publish workflow pushed: sha-<7 hex> (never moves), develop, master or a version. -# -# A deploy pulls both images first and changes nothing when a pull fails. It then tags the images -# that run now as ":rollback", writes the tag to .env, recreates the containers without building -# and waits until the backend and the frontend answer on their loopback ports and, for an image -# that renders on the server, until a page arrives rendered. When they do not, it switches back -# to the ":rollback" images by itself and exits 1. -# -# The database and the volumes are never changed by this script. Before a deploy it takes a dump of -# the database, because a rollback switches images only: it does not undo a migration that the new -# backend applied on start, and that dump is the way back. -# -# Environment: -# PKGDIFF_GHCR_TOKEN_FILE file with a read-only GHCR token (default ~/.pkgdiff-ghcr-token); -# when it exists, the script logs in to ghcr.io before it pulls -# PKGDIFF_GHCR_USER GitHub user the token belongs to (default fleizean) -# PKGDIFF_HEALTH_TIMEOUT seconds to wait for a healthy stack (default 180) -# PKGDIFF_BACKUP_DIR where the dumps go (default ~/pkgdiff-backups) -# PKGDIFF_BACKUP_KEEP how many dumps to keep (default 10) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -43,7 +25,6 @@ LEGACY_BACKEND_URL="${PKGDIFF_LEGACY_BACKEND_URL:-http://127.0.0.1:8080/api/pack FRONTEND_URL="${PKGDIFF_FRONTEND_URL:-http://127.0.0.1:8090/}" HEALTH_TIMEOUT="${PKGDIFF_HEALTH_TIMEOUT:-180}" HEALTH_INTERVAL="${PKGDIFF_HEALTH_INTERVAL:-5}" -# What Angular writes on the root element of a page it rendered on the server RENDERED_MARKER='ng-server-context="ssr"' BACKUP_DIR="${PKGDIFF_BACKUP_DIR:-$HOME/pkgdiff-backups}" @@ -69,7 +50,6 @@ revision() { docker image inspect "$1" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true } -# Image ID that the container of a service runs; empty when there is no such container. running_image() { docker inspect "pkgdiff-$1" --format '{{.Image}}' 2>/dev/null || true; } # The shell value wins over the one in .env, so compose sees the tag before .env is written. @@ -86,7 +66,6 @@ configured_tag() { sed -n 's/^PKGDIFF_IMAGE_TAG=//p' "$ENV_FILE" | tail -n 1 | tr -d '\r"'"'" } -# Replaces (or appends) the one PKGDIFF_IMAGE_TAG line; every other line of .env stays as it is. write_tag() { local tag="$1" tmp tmp="$(mktemp "$ENV_FILE.XXXXXX")" || return 1 @@ -122,9 +101,7 @@ preflight() { fi } -# The backend image runs as an unprivileged user and compose mounts the settings file into it. -# A file only root can read would stop the new backend from starting, so this is settled before -# anything changes: the file is given to the group of that user (it stays unreadable for others). +# The non-root backend user must read the mounted settings file; settle that before any change. settings_readable() { local backend user backend="$(image backend "$1")" @@ -147,9 +124,7 @@ login() { die "login to ghcr.io failed with the token in $TOKEN_FILE; nothing was changed" } -# Dumps the database next to the earlier dumps and keeps the newest ones. A dump that cannot be -# taken or is not complete stops the deploy before anything was changed. Without a running -# database container (a first install) there is nothing to dump. +# A rollback does not undo migrations, so this dump is the way back; a bad dump aborts the deploy. backup_database() { local tag="$1" file old if [ -z "$(running_image db)" ]; then @@ -167,7 +142,6 @@ backup_database() { rm -f "$file.partial" die "the database backup failed; nothing was changed" fi - # The names start with the time of the dump, so their order is the order in time ls -1 "$BACKUP_DIR"/pkgdiffdb-*.sql.gz 2>/dev/null | sort -r | tail -n "+$((BACKUP_KEEP + 1))" | while IFS= read -r old; do rm -f "$old" done @@ -175,8 +149,7 @@ backup_database() { http_ok() { curl -fsS -o /dev/null --max-time 10 "$1" 2>/dev/null; } -# Ready means the database and Redis answer too. An image from before /health/ready existed answers -# 404 there and is asked through its old health endpoint, so rolling back to it still works. +# Older images answer 404 on /health/ready; fall back to their old endpoint so rollback works. backend_ok() { local code code="$(curl -s -o /dev/null --max-time 10 -w '%{http_code}' "$BACKEND_URL" 2>/dev/null || true)" @@ -187,14 +160,11 @@ backend_ok() { esac } -# An image that renders its pages on the server says so in a label; an older one serves the -# client-side application only. renders_on_server() { [ "$(docker image inspect "$1" --format '{{index .Config.Labels "dev.pkgdiff.rendering"}}' 2>/dev/null || true)" = 'server' ] } -# nginx answers with the client-side page while the render server is down, so a 200 alone says -# nothing about it: the page itself has to say that it was rendered. +# nginx serves the client-side page while the render server is down, so a 200 proves nothing. page_is_rendered() { curl -fsS --max-time 20 "$FRONTEND_URL" 2>/dev/null | grep -q "$RENDERED_MARKER"; } # Both ports must stay on the loopback interface: the host nginx is the only public entry. @@ -203,7 +173,6 @@ ports_ok() { [ "$(docker port pkgdiff-frontend 80/tcp 2>/dev/null)" = '127.0.0.1:8090' ] } -# Waits until the containers run the images of this tag and answer; prints the reason when they do not. wait_healthy() { local tag="$1" deadline=$((SECONDS + HEALTH_TIMEOUT)) s wanted problem while :; do @@ -240,8 +209,7 @@ have_rollback_point() { done } -# The tag to write when going back: the earlier tag name while it still names the rollback -# images, otherwise the local ":rollback" tag (a moving tag such as develop has moved on). +# A moving tag may no longer name the rollback images; then the local ":rollback" tag is used. rollback_tag_name() { local previous="$1" s if [ -n "$previous" ] && [ "$previous" != "$ROLLBACK_TAG" ] && valid_tag "$previous"; then @@ -257,7 +225,6 @@ rollback_tag_name() { printf '%s' "$ROLLBACK_TAG" } -# Remembers what runs now, image and tag name, unless this deploy is a repeat of the last one. mark_rollback_point() { local tag="$1" previous="$2" s running changed=0 for s in $SERVICES; do diff --git a/deploy/grafana/provisioning/dashboards/pkgdiff.yml b/deploy/grafana/provisioning/dashboards/pkgdiff.yml index 35334d0..9a2f66f 100644 --- a/deploy/grafana/provisioning/dashboards/pkgdiff.yml +++ b/deploy/grafana/provisioning/dashboards/pkgdiff.yml @@ -1,4 +1,4 @@ -# Dashboards from deploy/grafana/dashboards, read-only in Grafana: a change belongs in the repository +# Read-only in Grafana: dashboards are changed in the repository apiVersion: 1 providers: diff --git a/deploy/grafana/provisioning/datasources/prometheus.yml b/deploy/grafana/provisioning/datasources/prometheus.yml index 28927d1..d37720b 100644 --- a/deploy/grafana/provisioning/datasources/prometheus.yml +++ b/deploy/grafana/provisioning/datasources/prometheus.yml @@ -1,5 +1,4 @@ -# The Prometheus of this compose stack. Its own name and uid, so it never collides with a data -# source that was created by hand before. +# Own name and uid so it never collides with a data source created by hand. apiVersion: 1 datasources: diff --git a/deploy/nginx/cloudflare-realip.conf b/deploy/nginx/cloudflare-realip.conf index a3c94df..2a21793 100644 --- a/deploy/nginx/cloudflare-realip.conf +++ b/deploy/nginx/cloudflare-realip.conf @@ -1,12 +1,5 @@ -# Host nginx — /etc/nginx/conf.d/cloudflare-realip.conf -# -# The site is behind Cloudflare, so every connection reaches the host nginx from a Cloudflare -# address. This file makes $remote_addr the visitor's address again (CF-Connecting-IP), but only -# for connections that really come from Cloudflare. pkgdiff.conf forwards $remote_addr to the -# containers, which is what rate limiting, IP blocking and the request log use. -# -# The ranges are Cloudflare's published list. Compare with https://www.cloudflare.com/ips-v4 and -# https://www.cloudflare.com/ips-v6 before applying, then: nginx -t && systemctl reload nginx +# Host nginx (/etc/nginx/conf.d): restores the visitor IP on connections from Cloudflare. +# Ranges from https://www.cloudflare.com/ips-v4 and https://www.cloudflare.com/ips-v6. set_real_ip_from 173.245.48.0/20; set_real_ip_from 103.21.244.0/22; diff --git a/deploy/nginx/pkgdiff.conf b/deploy/nginx/pkgdiff.conf index 92d4c57..ec617a8 100644 --- a/deploy/nginx/pkgdiff.conf +++ b/deploy/nginx/pkgdiff.conf @@ -1,30 +1,12 @@ -# Host nginx config — /etc/nginx/sites-available/pkgdiff -# Certificates already exist on the host: /etc/letsencrypt/live/pkgdiff.dev/ -# Apply changes: copy this file over the one on the host, then `nginx -t && systemctl reload nginx`. -# Behind Cloudflare, cloudflare-realip.conf (same folder) must be installed as well: without it -# $remote_addr below is a Cloudflare address and every visitor shares one rate-limit bucket. -# -# Every HTTPS name only answers Cloudflare (Authenticated Origin Pulls): a direct request to the -# server address would skip Cloudflare Access, which guards /admin, /api/admin, /api/auth and -# /hangfire. Turn on SSL/TLS > Origin Server > Authenticated Origin Pulls > Global in Cloudflare -# BEFORE reloading this file, and install Cloudflare's client CA once: -# curl -fsSo /etc/nginx/cloudflare-origin-pull-ca.pem \ -# https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem -# Turning that switch off while these lines are live answers every visitor with 400. -# -# "http2 on;" needs nginx 1.25.1 or newer. On an older nginx the runbook installs this file with -# the older spelling ("listen 443 ssl http2;"), which is the only one that version knows. -# -# Headers: this layer terminates TLS, so it sends Strict-Transport-Security, on every answer of -# every name. The other security headers of pkgdiff.dev come from the container nginx behind it, -# which knows the application; api.pkgdiff.dev goes straight to the backend and gets them here. +# Host nginx (/etc/nginx/sites-available/pkgdiff); needs cloudflare-realip.conf installed too. +# HTTPS only answers Cloudflare (Authenticated Origin Pulls) so Cloudflare Access can't be bypassed. +# "http2 on;" needs nginx 1.25.1 or newer. map $http_upgrade $connection_upgrade { default upgrade; '' close; } -# HTTP → HTTPS server { listen 80; server_name pkgdiff.dev www.pkgdiff.dev api.pkgdiff.dev; @@ -32,7 +14,6 @@ server { return 301 https://$host$request_uri; } -# www → non-www server { listen 443 ssl; http2 on; @@ -51,7 +32,6 @@ server { return 301 https://pkgdiff.dev$request_uri; } -# Frontend (container nginx: 8090 → files, rendered pages, /api, /analysisHub, /hangfire) server { listen 443 ssl; http2 on; @@ -67,10 +47,9 @@ server { ssl_session_cache shared:SSL:10m; ssl_session_timeout 10m; - # The largest request is a blog post saved from the admin panel client_max_body_size 2m; - # The backend sends this header too once it sees an HTTPS request; one copy is enough + # The backend sends HSTS too; keep only one copy proxy_hide_header Strict-Transport-Security; add_header Strict-Transport-Security "max-age=31536000" always; @@ -89,7 +68,6 @@ server { } } -# API subdomain — straight to the backend container server { listen 443 ssl; http2 on; @@ -110,8 +88,6 @@ server { add_header X-Content-Type-Options "nosniff" always; add_header X-Frame-Options "DENY" always; add_header Referrer-Policy "no-referrer" always; - # This name answers API calls with JSON and files; nothing it sends is a page that may load - # anything. The Hangfire dashboard is used through pkgdiff.dev/hangfire, not through this name. add_header Content-Security-Policy "default-src 'none'; frame-ancestors 'none'" always; location / { diff --git a/deploy/remote.sh b/deploy/remote.sh index 163674c..9f5d251 100644 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -6,12 +6,7 @@ # remote.sh rollback run the images again that ran before the last deploy # remote.sh status show the configured tag and what is running # -# It is meant to be the forced command of that key in ~/.ssh/authorized_keys (docs/ops/deploy-ghcr.md): -# -# command="/opt/pkgdiff/src/deploy/remote.sh",no-pty,no-port-forwarding,no-agent-forwarding,no-X11-forwarding ssh-ed25519 AAAA... pkgdiff-actions-deploy -# -# sshd then runs this script whatever the client asked for and hands over what it asked for in -# SSH_ORIGINAL_COMMAND. That text is only split into words and checked; nothing in it is evaluated. +# Forced command of the deploy key: SSH_ORIGINAL_COMMAND is split and checked, never evaluated. set -euo pipefail usage() { @@ -62,11 +57,9 @@ main() { cd "$root" if [ "$action" = deploy ]; then - # The compose file and the deploy script belong to the commit the images were built from. - # A fast-forward only: a checkout with local commits or changes in the way stops here. if [ -n "$commit" ]; then git fetch --quiet origin - # Only what was merged: a commit that is on no deployed branch is refused, whoever asks + # Refuse a commit that is on no deployed branch if ! git merge-base --is-ancestor "$commit" origin/develop 2>/dev/null \ && ! git merge-base --is-ancestor "$commit" origin/master 2>/dev/null; then echo "deploy: $commit is not on develop or master; nothing was done" >&2 diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 13ebdc3..ed763c0 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -1,15 +1,14 @@ -# Builds backend and frontend from this checkout instead of pulling them from GHCR. -# Always used together with the main file, with PKGDIFF_IMAGE_TAG=local in .env: +# Builds backend and frontend from source; use with PKGDIFF_IMAGE_TAG=local in .env: # docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build services: backend: build: context: ./src/backend dockerfile: Dockerfile - # A local build runs on the defaults in the image and on .env; no server settings file is mounted + # Local builds mount no server settings file volumes: !reset [] - # The ssr service runs this same image; it is built once, here + # The ssr service runs this same image frontend: build: context: ./src/frontend diff --git a/docker-compose.yml b/docker-compose.yml index 390dfa1..7ddfb7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,5 @@ -# Production stack. Backend and frontend run prebuilt images from GHCR; nothing is built here. -# Deploy with deploy/deploy.sh (docs/ops/deploy-ghcr.md). To build from source instead, add -# docker-compose.build.yml: docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build -# -# Every service has a memory and a process limit, so one of them running away cannot take the -# host (which also runs other projects) down with it; the two that depend on load can be changed -# in .env. Nothing but the two ports the host nginx proxies to and the two monitoring ports is -# published, and all four on the loopback interface only. +# Production stack: backend and frontend run prebuilt GHCR images, deployed by deploy/deploy.sh. +# To build from source, add -f docker-compose.build.yml. services: backend: image: ghcr.io/pkgdiff-dev/pkgdiff-backend:${PKGDIFF_IMAGE_TAG:?PKGDIFF_IMAGE_TAG is not set; deploy/deploy.sh writes it to .env} @@ -28,20 +22,16 @@ services: - AdminSettings__ViewerPassword=${VIEWER_PASSWORD:-} - ForwardedHeaders__TrustedNetworks=${FORWARDED_TRUSTED_NETWORKS:-172.28.0.0/16} - GitHubSettings__PersonalAccessToken=${GITHUB_PAT:-} - # The image runs as an unprivileged user (uid 1654) and needs no capability cap_drop: - ALL security_opt: - no-new-privileges:true - # No CPU limit: the work of the backend is bounded in the application (10 analyses at a time, - # 10 job workers), and a limit above the number of cores of the host would stop the container - # from being created at all + # No CPU limit: one above the host's core count stops the container from being created mem_limit: ${PKGDIFF_BACKEND_MEMORY:-2g} pids_limit: 1024 ports: - "127.0.0.1:8080:8080" volumes: - # The settings of this server, on top of the defaults in the image (appsettings.json, no secrets) - ./config/appsettings.json:/app/appsettings.Production.json:ro depends_on: db: @@ -49,8 +39,7 @@ services: redis: condition: service_healthy healthcheck: - # Ready means the database and Redis answer. An image from before /health/ready existed answers 404 there - # and is asked through its old health endpoint, so a rollback to it still shows as healthy. + # Older images answer 404 on /health/ready; fall back to their old endpoint so rollback works. test: ["CMD-SHELL", "code=$$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/health/ready); [ \"$$code\" = 200 ] || { [ \"$$code\" = 404 ] && curl -fsS -o /dev/null http://localhost:8080/api/package/health; }"] interval: 30s timeout: 10s @@ -65,14 +54,11 @@ services: max-file: "3" - # The render server: the frontend image run with Node instead of nginx. It renders the public - # pages and asks the API through the frontend container, the way a browser does. ssr: image: ghcr.io/pkgdiff-dev/pkgdiff-frontend:${PKGDIFF_IMAGE_TAG:?PKGDIFF_IMAGE_TAG is not set; deploy/deploy.sh writes it to .env} container_name: pkgdiff-ssr restart: always - # An image from before server rendering has no render server. The service then idles, so a - # rollback to such an image still brings the stack up; its nginx never asks this service. + # Older images have no render server; idle instead so a rollback still brings the stack up. entrypoint: ["/bin/sh", "-c", "if [ -f /app/server/server.mjs ]; then exec node /app/server/server.mjs; else echo 'this image has no render server; idling'; exec sleep infinity; fi"] user: "65534:65534" read_only: true @@ -85,7 +71,6 @@ services: environment: - NODE_ENV=production - PORT=4000 - # Where a render finds the API: nginx of the frontend service, inside the compose network - SSR_INTERNAL_ORIGIN=http://frontend mem_limit: 512m cpus: 1.0 @@ -109,9 +94,7 @@ services: image: ghcr.io/pkgdiff-dev/pkgdiff-frontend:${PKGDIFF_IMAGE_TAG:?PKGDIFF_IMAGE_TAG is not set; deploy/deploy.sh writes it to .env} container_name: pkgdiff-frontend restart: always - # The image runs nginx as the unprivileged "nginx" user. Capabilities and the file system - # are left as they are on purpose: an image from before that change (a rollback) starts - # nginx as root, which still needs both. + # No cap_drop/read_only on purpose: older images (a rollback) still start nginx as root. security_opt: - no-new-privileges:true mem_limit: 256m @@ -120,14 +103,13 @@ services: ports: - "127.0.0.1:8090:80" healthcheck: - # A file nginx serves itself: its health says nothing about the render server or the backend. # 127.0.0.1, not localhost: localhost resolves to ::1 first and nginx listens on IPv4 only. test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:80/robots.txt || exit 1"] interval: 30s timeout: 10s retries: 3 start_period: 20s - # nginx looks the backend up once, when it starts; the render server is looked up per request + # nginx resolves the backend once at start; the render server is resolved per request depends_on: backend: condition: service_healthy @@ -150,13 +132,11 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=pkgdiffdb - # The entrypoint starts as root to fix the ownership of the data directory and then runs the - # server as the "postgres" user + # No cap_drop: the entrypoint starts as root to chown the data directory security_opt: - no-new-privileges:true mem_limit: ${PKGDIFF_DB_MEMORY:-1024m} pids_limit: 512 - # No port on the host: backups and maintenance use docker exec volumes: - postgres-data:/var/lib/postgresql networks: @@ -183,11 +163,9 @@ services: mem_limit: 320m cpus: 1.0 pids_limit: 128 - # No port on the host: only the backend reaches it, over the compose network volumes: - redis-data:/data - # Redis holds the cache and the rate-limit counters, nothing that cannot be rebuilt: at its - # own limit, below the limit of the container, it drops the entries used least recently + # Only rebuildable data, so evict LRU below the container memory limit command: redis-server --stop-writes-on-bgsave-error no --maxmemory 192mb --maxmemory-policy allkeys-lru networks: - pkgdiff-network @@ -214,7 +192,6 @@ services: mem_limit: 512m cpus: 0.5 pids_limit: 256 - # Loopback only: reached from outside through an SSH tunnel (docs/ops/operations.md) ports: - "127.0.0.1:${PKGDIFF_PROMETHEUS_PORT:-9390}:9090" volumes: @@ -258,16 +235,13 @@ services: mem_limit: 512m cpus: 0.5 pids_limit: 256 - # Loopback only: reached from outside through an SSH tunnel (docs/ops/operations.md) ports: - "127.0.0.1:${PKGDIFF_GRAFANA_PORT:-3300}:3000" volumes: - grafana-data:/var/lib/grafana - # The Prometheus data source and the pkgdiff dashboard, from this repository - ./deploy/grafana/provisioning:/etc/grafana/provisioning:ro - ./deploy/grafana/dashboards:/var/lib/grafana/dashboards:ro - # No health condition on Prometheus: monitoring must never be the reason an application - # deploy fails + # No health condition: monitoring must never be the reason an application deploy fails depends_on: - prometheus healthcheck: diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 9be1bcd..6ebbc75 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -1,12 +1,10 @@ -# Build stage: the SDK image that generated the lock files (same reference as in Directory.Build.props) +# The SDK that generated the lock files; must match the reference in Directory.Build.props FROM mcr.microsoft.com/dotnet/sdk:10.0.402-noble@sha256:0d99ebf1ccad15037b5c98c9fe8979d6b8dbac23812feae2277a0b277b5c5ffa AS build # Emergency only: --build-arg PKGDIFF_AUDIT_AS_ERRORS=false turns NU1901-NU1904 back into warnings ARG PKGDIFF_AUDIT_AS_ERRORS=true WORKDIR /src -# Central versions, audit settings, the package source and the lock files come first, -# so the restore layer stays cached until a dependency changes COPY ["PkgDiff.slnx", "Directory.Packages.props", "Directory.Build.props", "nuget.config", "./"] COPY ["PkgDiff.API/PkgDiff.csproj", "PkgDiff.API/packages.lock.json", "PkgDiff.API/"] COPY ["PkgDiff.Application/PkgDiff.Application.csproj", "PkgDiff.Application/packages.lock.json", "PkgDiff.Application/"] @@ -14,29 +12,21 @@ COPY ["PkgDiff.Domain/PkgDiff.Domain.csproj", "PkgDiff.Domain/packages.lock.json COPY ["PkgDiff.Infrastructure/PkgDiff.Infrastructure.csproj", "PkgDiff.Infrastructure/packages.lock.json", "PkgDiff.Infrastructure/"] COPY ["PkgDiff.Shared/PkgDiff.Shared.csproj", "PkgDiff.Shared/packages.lock.json", "PkgDiff.Shared/"] -# Restores exactly what the lock files record. NuGet audit runs inside this restore, so a -# known-vulnerable package fails the build here, but only when this layer is rebuilt (cold cache -# or a changed manifest). .github/scripts/check-deps.sh is the check that audits on every run. +# The audit here only runs when this layer rebuilds; check-deps.sh audits on every run. RUN dotnet restore "PkgDiff.API/PkgDiff.csproj" --locked-mode -p:ContinuousIntegrationBuild=true -p:PkgDiffAuditAsErrors=${PKGDIFF_AUDIT_AS_ERRORS} COPY . . -# Publish reuses the restore above, so nothing is resolved a second time WORKDIR /src/PkgDiff.API RUN dotnet publish "PkgDiff.csproj" -c Release -o /app/publish --no-restore /p:UseAppHost=false -p:PkgDiffAuditAsErrors=${PKGDIFF_AUDIT_AS_ERRORS} -# Runtime stage: ASP.NET Core 10.0.13 on Ubuntu 24.04, pinned by tag and index digest FROM mcr.microsoft.com/dotnet/aspnet:10.0.13-noble@sha256:d5633778c754e89a386882eddf8dbee6942447a8a4b8267c19cf1e1158d0b0ac AS final WORKDIR /app EXPOSE 8080 -# Pick up Ubuntu security fixes published after the runtime image was built, and install curl, -# which the compose healthcheck calls. -# The base digest is pinned, so Docker would reuse this layer from its build cache -# forever; bump the date (or pass --build-arg APT_UPGRADE_DATE=...) to run it again. +# With a pinned base this layer stays cached forever; bump the date to pick up security fixes. +# curl is needed by the compose healthcheck. ARG APT_UPGRADE_DATE=2026-10-03 -# Non-interactive, and a changed configuration file keeps the installed version: an upgrade must -# never stop the image build at a question. The variable is set for this layer only, not as ENV. RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get update \ && apt-get upgrade -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold \ @@ -45,8 +35,5 @@ RUN export DEBIAN_FRONTEND=noninteractive \ COPY --from=build /app/publish . -# The unprivileged user the ASP.NET image ships (uid 1654). The application writes nothing next -# to its files; a settings file mounted by compose has to be readable for this user -# (deploy/deploy.sh checks that before it switches). USER $APP_UID ENTRYPOINT ["dotnet", "PkgDiff.dll"] diff --git a/src/backend/PkgDiff.API/Controllers/Admin/AdminDashboardController.cs b/src/backend/PkgDiff.API/Controllers/Admin/AdminDashboardController.cs index b628563..5f43b13 100644 --- a/src/backend/PkgDiff.API/Controllers/Admin/AdminDashboardController.cs +++ b/src/backend/PkgDiff.API/Controllers/Admin/AdminDashboardController.cs @@ -19,8 +19,6 @@ public AdminDashboardController(IAdminDashboardService service, ILogger> GetDashboard() @@ -37,8 +35,6 @@ public async Task> GetDashboard() } } - // ───── Search History ───── - [HttpGet("search-history")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Security.ViewLogs)] public async Task>> GetSearchHistory( diff --git a/src/backend/PkgDiff.API/Controllers/Admin/AdminPackagesController.cs b/src/backend/PkgDiff.API/Controllers/Admin/AdminPackagesController.cs index abd5312..4a66920 100644 --- a/src/backend/PkgDiff.API/Controllers/Admin/AdminPackagesController.cs +++ b/src/backend/PkgDiff.API/Controllers/Admin/AdminPackagesController.cs @@ -19,8 +19,6 @@ public AdminPackagesController(IAdminPackageService service, ILogger>> GetPackages( @@ -61,8 +59,6 @@ public async Task DeletePackage(int id) } } - // ───── Registries ───── - [HttpGet("registries")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Registries.View)] public async Task>> GetRegistries() diff --git a/src/backend/PkgDiff.API/Controllers/Admin/AdminSecurityController.cs b/src/backend/PkgDiff.API/Controllers/Admin/AdminSecurityController.cs index 4d64ee1..046e43c 100644 --- a/src/backend/PkgDiff.API/Controllers/Admin/AdminSecurityController.cs +++ b/src/backend/PkgDiff.API/Controllers/Admin/AdminSecurityController.cs @@ -95,7 +95,7 @@ public async Task UnblockIp(int id) } } - // Both are required: the seeded Viewer role holds ViewLogs, and the only consumer (IP Blocking page) is gated on ManageIps. + // ViewLogs alone is too broad (the seeded Viewer role has it), so ManageIps is required too [HttpGet("security/ip-statistics")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Security.ViewLogs)] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Security.ManageIps)] diff --git a/src/backend/PkgDiff.API/Controllers/AdminBlogController.cs b/src/backend/PkgDiff.API/Controllers/AdminBlogController.cs index 07c239c..09801ea 100644 --- a/src/backend/PkgDiff.API/Controllers/AdminBlogController.cs +++ b/src/backend/PkgDiff.API/Controllers/AdminBlogController.cs @@ -113,7 +113,6 @@ public async Task UpdateBlog(int id, [FromBody] UpdateBlogRequest return Ok(blog); } - // The editor previews through the same renderer the public blog page uses [HttpPost("preview")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Blog.Manage)] public IActionResult PreviewBlog([FromBody] BlogPreviewRequest request) diff --git a/src/backend/PkgDiff.API/Controllers/AuditController.cs b/src/backend/PkgDiff.API/Controllers/AuditController.cs index f03b3fb..602bc54 100644 --- a/src/backend/PkgDiff.API/Controllers/AuditController.cs +++ b/src/backend/PkgDiff.API/Controllers/AuditController.cs @@ -4,13 +4,11 @@ namespace PkgDiff.API.Controllers; -// Dependency audit: a visitor pastes a dependency file, checks the packages read from it and gets a -// report of their known vulnerabilities. The report is made by a background job and polled here. [ApiController] [Route("api/[controller]")] public class AuditController : ControllerBase { - // Audits waiting or running at the same time, all visitors together; each one takes a job worker + // All visitors together; each audit takes a job worker private const int MaxActiveAudits = 20; private const long MaxRequestBytes = 2 * 1024 * 1024; @@ -25,7 +23,6 @@ public AuditController(IManifestParser parser, IDependencyAuditService audits, I _logger = logger; } - // Reads the packages out of the file; nothing is fetched and nothing is stored [HttpPost("parse")] [RequestSizeLimit(MaxRequestBytes)] public ActionResult Parse([FromBody] ManifestParseRequest request) diff --git a/src/backend/PkgDiff.API/Controllers/BlogController.cs b/src/backend/PkgDiff.API/Controllers/BlogController.cs index 9959f13..7147ac8 100644 --- a/src/backend/PkgDiff.API/Controllers/BlogController.cs +++ b/src/backend/PkgDiff.API/Controllers/BlogController.cs @@ -35,7 +35,6 @@ public async Task GetPublishedBlogs([FromQuery] int page = 1, [Fr b.PackageId, b.Version, b.PublishedAt, - // Only send a small snippet of the content for the list view Excerpt = b.Content.Length > 200 ? b.Content.Substring(0, 200) + "..." : b.Content }, page, diff --git a/src/backend/PkgDiff.API/Controllers/ContributorsController.cs b/src/backend/PkgDiff.API/Controllers/ContributorsController.cs index af7d15b..7bef6d1 100644 --- a/src/backend/PkgDiff.API/Controllers/ContributorsController.cs +++ b/src/backend/PkgDiff.API/Controllers/ContributorsController.cs @@ -18,9 +18,6 @@ public ContributorsController(IContributorService contributorService, IJobSchedu _jobs = jobs; } - /// - /// Returns the top contributors leaderboard ranked by GitHub @mention count. - /// [HttpGet] public async Task GetLeaderboard([FromQuery] int limit = 100) { @@ -45,9 +42,6 @@ public async Task GetLeaderboardPaged([FromQuery] int page = 1, [ }); } - /// - /// Triggers a one-time backfill job to scan ALL existing release notes and populate Contributors table. - /// [HttpPost("backfill")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.Contributors.Manage)] public IActionResult TriggerBackfill() diff --git a/src/backend/PkgDiff.API/Controllers/HealthController.cs b/src/backend/PkgDiff.API/Controllers/HealthController.cs index f0734bb..fc1980b 100644 --- a/src/backend/PkgDiff.API/Controllers/HealthController.cs +++ b/src/backend/PkgDiff.API/Controllers/HealthController.cs @@ -16,9 +16,6 @@ public HealthController(IHealthCheckService healthCheckService) _healthCheckService = healthCheckService; } - /// - /// Get overall health status (quick check) - /// [HttpGet("status")] public async Task> GetStatus() { @@ -26,9 +23,6 @@ public async Task> GetStatus() return Ok(status); } - /// - /// Get detailed health information for all services - /// [HttpGet("details")] public async Task> GetDetails() { diff --git a/src/backend/PkgDiff.API/Controllers/PackageController.cs b/src/backend/PkgDiff.API/Controllers/PackageController.cs index f931da5..8b4fec5 100644 --- a/src/backend/PkgDiff.API/Controllers/PackageController.cs +++ b/src/backend/PkgDiff.API/Controllers/PackageController.cs @@ -16,13 +16,10 @@ public class PackageController : ControllerBase { private const int MaxConcurrentAnalyses = 10; - // One visitor must not take every analysis slot: a few at once per address private const int MaxConcurrentAnalysesPerAddress = 2; private static readonly System.Collections.Concurrent.ConcurrentDictionary AnalysesPerAddress = new(); - // Opening a package nobody looked at before queues a scrape of its notes. Asking for one new name after - // another must not fill the job queue or use up the GitHub budget: scrapes queued this way are capped - // per address and in total, per hour. A package over the cap is still shown, and scraped on a later visit. + // Caps first-visit scrapes so a stream of new names cannot fill the queue or the GitHub budget private const int MaxScrapesPerAddressPerHour = 30; private const int MaxScrapesPerHour = 600; private const int ClientClosedRequest = 499; @@ -129,7 +126,7 @@ private async Task> AnalyzeAsync(AnalyzeRequest req { var ipAddress = HttpContext.GetClientIpAddress(); - // When this request is aborted and nobody else waits for the same analysis, the work behind it stops + // Aborting the request stops the work unless another request waits on the same analysis using var work = WorkCancellation.Use(cancellationToken); var result = await _analysisService.AnalyzePackageAsync(request, ipAddress); await MarkViewedAsync(request.Registry, request.PackageId); @@ -211,7 +208,6 @@ public async Task> GetPackageMetadata(string regis var metadata = WithTrust(await _analysisService.GetPackageMetadataAsync(registry, packageId), await TrustOfAsync(registry, packageId)); await MarkViewedAsync(registry, packageId); - // Trigger background scraping if we have unscraped versions and no job is already running var hasUnscrapedVersions = metadata.Versions.Any(v => string.IsNullOrEmpty(v.ReleaseNotesSource)); var cacheKey = $"scrape-attempt-{registry}-{packageId}"; @@ -277,7 +273,6 @@ public async Task>> GetCachedPackages() { try { - // The same list for every visitor: built at most every few minutes var cachedPackages = await _memory.GetOrCreateAsync("cached-packages", entry => { entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5); @@ -306,7 +301,6 @@ public async Task ExportReleaseNotesToPdf(string registry, string { var metadata = WithTrust(await _analysisService.GetPackageMetadataAsync(registry, packageId), await TrustOfAsync(registry, packageId)); - // Generate PDF var fileContent = await _exportService.ExportToPdfAsync(metadata); var fileName = $"{packageId}-release-notes.pdf"; @@ -336,13 +330,11 @@ public async Task ExportDiffToPdf([FromBody] AnalyzeRequest reque try { - // Perform analysis var ipAddress = HttpContext.GetClientIpAddress(); using var work = WorkCancellation.Use(cancellationToken); var diff = await _analysisService.AnalyzePackageAsync(request, ipAddress); diff.Trust = await TrustOfAsync(request.Registry, request.PackageId); - // Generate PDF var fileContent = await _exportService.ExportDiffToPdfAsync(diff); var fileName = $"{request.PackageId}-diff-{request.CurrentVersion}-to-{request.TargetVersion}.pdf"; diff --git a/src/backend/PkgDiff.API/Controllers/SitemapController.cs b/src/backend/PkgDiff.API/Controllers/SitemapController.cs index b96de5f..51078a1 100644 --- a/src/backend/PkgDiff.API/Controllers/SitemapController.cs +++ b/src/backend/PkgDiff.API/Controllers/SitemapController.cs @@ -28,7 +28,6 @@ public async Task GetDynamicSitemap() xml.AppendLine(""); xml.AppendLine(""); - // Static pages xml.AppendLine(" "); xml.AppendLine(" https://pkgdiff.dev/"); xml.AppendLine(" daily"); @@ -71,7 +70,6 @@ public async Task GetDynamicSitemap() xml.AppendLine($" {DateTime.UtcNow:yyyy-MM-dd}"); xml.AppendLine(" "); - // Package pages foreach (var package in packages) { var encodedPackageId = Uri.EscapeDataString(package.PackageId); diff --git a/src/backend/PkgDiff.API/Controllers/VersionExcludePatternsController.cs b/src/backend/PkgDiff.API/Controllers/VersionExcludePatternsController.cs index 517917b..70b0017 100644 --- a/src/backend/PkgDiff.API/Controllers/VersionExcludePatternsController.cs +++ b/src/backend/PkgDiff.API/Controllers/VersionExcludePatternsController.cs @@ -19,9 +19,6 @@ public VersionExcludePatternsController(IUnitOfWork unitOfWork, IVersionFilterSe _versionFilterService = versionFilterService; } - /// - /// Get all version exclude patterns - /// [HttpGet] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.View)] public async Task>> GetAll() @@ -30,9 +27,6 @@ public async Task>> GetAll() return Ok(patterns.OrderBy(p => p.Id)); } - /// - /// Get a specific pattern by ID - /// [HttpGet("{id}")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.View)] public async Task> GetById(int id) @@ -44,9 +38,6 @@ public async Task> GetById(int id) return Ok(pattern); } - /// - /// Create a new exclude pattern - /// [HttpPost] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.Create)] public async Task> Create([FromBody] CreatePatternRequest request) @@ -64,15 +55,11 @@ public async Task> Create([FromBody] CreateP await _unitOfWork.VersionExcludePatterns.AddAsync(pattern); await _unitOfWork.SaveChangesAsync(); - // Refresh cache await _versionFilterService.RefreshPatternsAsync(); return CreatedAtAction(nameof(GetById), new { id = pattern.Id }, pattern); } - /// - /// Update an existing pattern - /// [HttpPut("{id}")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.Edit)] public async Task> Update(int id, [FromBody] UpdatePatternRequest request) @@ -93,15 +80,11 @@ public async Task> Update(int id, [FromBody] await _unitOfWork.VersionExcludePatterns.UpdateAsync(pattern); await _unitOfWork.SaveChangesAsync(); - // Refresh cache await _versionFilterService.RefreshPatternsAsync(); return Ok(pattern); } - /// - /// Delete a pattern - /// [HttpDelete("{id}")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.Delete)] public async Task Delete(int id) @@ -113,15 +96,11 @@ public async Task Delete(int id) await _unitOfWork.VersionExcludePatterns.DeleteAsync(pattern); await _unitOfWork.SaveChangesAsync(); - // Refresh cache await _versionFilterService.RefreshPatternsAsync(); return NoContent(); } - /// - /// Toggle pattern active status - /// [HttpPatch("{id}/toggle")] [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.Edit)] public async Task> ToggleActive(int id) @@ -134,24 +113,18 @@ public async Task> ToggleActive(int id) await _unitOfWork.VersionExcludePatterns.UpdateAsync(pattern); await _unitOfWork.SaveChangesAsync(); - // Refresh cache await _versionFilterService.RefreshPatternsAsync(); return Ok(pattern); } - /// - /// Preview: Test pattern against sample versions - /// Returns 3 example versions that would be excluded - /// [HttpPost("preview")] - [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.View)] // View might be enough, but it's a tool for editing + [Authorize(Policy = PkgDiff.Shared.Constants.Permissions.VersionExcludePatterns.View)] public ActionResult Preview([FromBody] PreviewRequest request) { if (string.IsNullOrWhiteSpace(request.Pattern)) return BadRequest(new { message = "Pattern is required" }); - // Sample versions for testing var sampleVersions = new[] { "1.0.0", @@ -192,7 +165,6 @@ public ActionResult Preview([FromBody] PreviewRequest request) } } -// DTOs public record CreatePatternRequest(string Pattern, string? Description, bool IsActive = true); public record UpdatePatternRequest(string? Pattern, string? Description, bool? IsActive); public record PreviewRequest(string Pattern); diff --git a/src/backend/PkgDiff.API/Extensions/AuthServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/AuthServiceCollectionExtensions.cs index c562a70..aa01fa2 100644 --- a/src/backend/PkgDiff.API/Extensions/AuthServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/AuthServiceCollectionExtensions.cs @@ -51,8 +51,7 @@ public static IServiceCollection AddPkgDiffAuthentication( IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(jwt.Value.SecretKey)) }; - // A signed token is not enough: its user must still exist, not be locked out and have the - // security stamp the token was issued with. The answer is cached for a minute per token stamp. + // A signed token is not enough: the user must exist, be unlocked and match the security stamp options.Events = new JwtBearerEvents { OnTokenValidated = async context => diff --git a/src/backend/PkgDiff.API/Extensions/DatabaseSeeder.cs b/src/backend/PkgDiff.API/Extensions/DatabaseSeeder.cs index a0c87d5..e735f6c 100644 --- a/src/backend/PkgDiff.API/Extensions/DatabaseSeeder.cs +++ b/src/backend/PkgDiff.API/Extensions/DatabaseSeeder.cs @@ -18,7 +18,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("DatabaseSeeder"); var configuration = scope.ServiceProvider.GetRequiredService(); - // Seed Roles if (!await roleManager.RoleExistsAsync("Admin")) { EnsureSucceeded(await roleManager.CreateAsync(new IdentityRole("Admin")), "create role Admin"); @@ -35,7 +34,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) await SeedAdminUserAsync(userManager, adminPassword, logger); - // Seed Viewer User (optional) var viewerUser = await userManager.FindByNameAsync("pkgdiff_viewer"); if (viewerUser == null && viewerPassword != null) { @@ -64,11 +62,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) logger.LogInformation("Viewer user not seeded: AdminSettings:ViewerPassword (env VIEWER_PASSWORD) is not set."); } - // ────────────────────────────── - // Seed Role Permissions - // ────────────────────────────── - - // 1. Admin Role (Grant ALL permissions) var adminRole = await roleManager.FindByNameAsync("Admin"); if (adminRole != null) { @@ -88,7 +81,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) if (newClaimsCount > 0) logger.LogInformation("Added {Count} new permissions to role {Role}", newClaimsCount, "Admin"); } - // 2. Viewer Role (Grant VIEW permissions) var viewerRole = await roleManager.FindByNameAsync("Viewer"); if (viewerRole != null) { @@ -121,9 +113,7 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) if (newClaimsCount > 0) logger.LogInformation("Added {Count} new permissions to role {Role}", newClaimsCount, "Viewer"); } - // Seed System Settings: insert missing known keys with their flags, repair flags of existing - // known keys and encrypt plaintext secrets. Idempotent: a second run changes nothing and never re-encrypts. - // The protector is resolved first: in Production a missing SETTINGS_ENCRYPTION_KEY stops startup here. + // Idempotent, never re-encrypts; the protector resolves first so a missing key stops startup var protector = scope.ServiceProvider.GetRequiredService(); var existingSettings = await context.SystemSettings.ToListAsync(); var createdSettings = SystemSettingsMaintenance.CreateMissing(existingSettings.Select(s => s.Key)); @@ -139,7 +129,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) logger.LogInformation("System settings: {Added} added, {Changed} updated", createdSettings.Count, changedSettings); } - // Seed Registries if (!await context.Registries.AnyAsync()) { var registries = new[] @@ -165,7 +154,6 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) } } - // Seed Version Exclude Patterns var expectedPatterns = new[] { new { Id = 1, Pattern = "dev", Description = "Development builds (e.g., 1.0.0-dev-123)" }, @@ -232,8 +220,7 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider) logger.LogInformation("Version exclude patterns seeded: {Added} added, {Updated} updated", addedCount, updatedCount); } } - // Bootstrap only: once any user holds Admin, a demoted or deleted pkgdiff_admin is an administrator's - // decision and is never re-promoted or recreated here. + // Bootstrap only: once any Admin exists, pkgdiff_admin is never re-promoted or recreated public static async Task SeedAdminUserAsync(UserManager userManager, string? adminPassword, ILogger logger) { if ((await userManager.GetUsersInRoleAsync("Admin")).Count > 0) @@ -267,7 +254,6 @@ public static async Task SeedAdminUserAsync(UserManager userMan logger.LogInformation("Admin user {UserName} created", "pkgdiff_admin"); } - // Reached only when no administrator exists at all (fresh install or an interrupted first seed) EnsureSucceeded(await userManager.AddToRoleAsync(adminUser, "Admin"), "assign role Admin to pkgdiff_admin"); logger.LogInformation("Role {Role} assigned to {UserName}", "Admin", "pkgdiff_admin"); } diff --git a/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs index 95364c0..951ebeb 100644 --- a/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs @@ -18,7 +18,6 @@ namespace PkgDiff.API.Extensions; -// The registrations that make up the host: each method is one concern of Program.cs public static class HostingServiceCollectionExtensions { public const string DevelopmentCorsPolicy = "AllowAngularDev"; @@ -41,7 +40,6 @@ public static IServiceCollection AddPkgDiffSwagger(this IServiceCollection servi BearerFormat = "JWT" }); - // Delegate-based API of Microsoft.OpenApi 2.x options.AddSecurityRequirement(document => new Microsoft.OpenApi.OpenApiSecurityRequirement { [new Microsoft.OpenApi.OpenApiSecuritySchemeReference("Bearer", document)] = [] @@ -73,8 +71,7 @@ public static IServiceCollection AddPkgDiffPersistence(this IServiceCollection s .AddEntityFrameworkStores() .AddDefaultTokenProviders(); - // Data Protection key ring in Postgres: dashboard tickets/cookies and progress tokens survive container recreation. - // Keys are encrypted with a key derived from SETTINGS_ENCRYPTION_KEY so a database dump alone cannot forge them. + // Kept in Postgres to survive recreation; encrypted so a database dump alone cannot forge tokens services.AddDataProtection() .SetApplicationName("pkgdiff") .PersistKeysToDbContext() @@ -99,7 +96,7 @@ public static IServiceCollection AddPkgDiffBackgroundJobs(this IServiceCollectio options.WorkerCount = 10; }); - // Dependency audits get two workers of their own and nothing else runs them + // Dependency audits get two dedicated workers services.AddHangfireServer(options => { options.ServerName = $"{Environment.MachineName}:audits"; @@ -112,14 +109,12 @@ public static IServiceCollection AddPkgDiffBackgroundJobs(this IServiceCollectio services.AddScoped(); services.AddScoped(); - // Hangfire dashboard access (ticket -> cookie, per-request Admin re-check) services.AddSingleton(); services.AddScoped(); return services; } - // Cache backend: CacheSettings:Backend is "Redis" or "Memory"; Redis also holds the rate-limit counters public static IServiceCollection AddPkgDiffCaching(this IServiceCollection services, IConfiguration configuration) { services.AddMemoryCache(); @@ -145,7 +140,6 @@ public static IServiceCollection AddPkgDiffCaching(this IServiceCollection servi services.AddSingleton(); } - // Inbound rate-limit tiers (RateLimiting section); the defaults apply when the section is absent services.AddOptions() .Bind(configuration.GetSection(InboundRateLimitOptions.SectionName)) .Validate(options => options.IsValid(), "RateLimiting: every PerMinute and PerFiveMinutes value must be greater than zero.") @@ -156,16 +150,15 @@ public static IServiceCollection AddPkgDiffCaching(this IServiceCollection servi public static IServiceCollection AddPkgDiffDiagnostics(this IServiceCollection services) { - // Any exception no action caught becomes a ProblemDetails response; the detail goes to the log only services.AddProblemDetails(); services.AddExceptionHandler(); - // /health is liveness (the process answers); /health/ready also needs the database and Redis + // /health is liveness; /health/ready also needs the database and Redis services.AddHealthChecks() .AddCheck("database", tags: new[] { HealthResponses.ReadyTag }) .AddCheck("redis", tags: new[] { HealthResponses.ReadyTag }); - // Bounded: when the database cannot keep up, the oldest pending log entries are dropped instead of memory growing + // Bounded: drops the oldest pending entries instead of growing memory when the DB lags services.AddSingleton(Channel.CreateBounded(new BoundedChannelOptions(10_000) { FullMode = BoundedChannelFullMode.DropOldest, diff --git a/src/backend/PkgDiff.API/Extensions/HttpContextExtensions.cs b/src/backend/PkgDiff.API/Extensions/HttpContextExtensions.cs index d494c86..5ca9145 100644 --- a/src/backend/PkgDiff.API/Extensions/HttpContextExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/HttpContextExtensions.cs @@ -2,8 +2,7 @@ namespace PkgDiff.API.Extensions; public static class HttpContextExtensions { - // RemoteIpAddress is already rewritten by UseForwardedHeaders for trusted proxies only, - // so request headers are never consulted here. + // Already rewritten by UseForwardedHeaders for trusted proxies; headers are never read here public static string GetClientIpAddress(this HttpContext context) { var ip = context.Connection.RemoteIpAddress; diff --git a/src/backend/PkgDiff.API/Extensions/OutboundHttpServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/OutboundHttpServiceCollectionExtensions.cs index 71bad56..ce415c6 100644 --- a/src/backend/PkgDiff.API/Extensions/OutboundHttpServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/OutboundHttpServiceCollectionExtensions.cs @@ -37,7 +37,7 @@ public static IServiceCollection AddPkgDiffOutboundHttp(this IServiceCollection client.DefaultRequestHeaders.Add("User-Agent", "PkgDiff/1.0"); }); - // Fixed trusted registry hosts get larger caps: real packuments (npm typescript ~16 MB) exceed 2 MiB. + // Trusted registry hosts get larger caps: real packuments exceed 2 MiB services.AddHttpClient(HttpClientNames.NpmRegistry, client => { client.BaseAddress = new Uri("https://registry.npmjs.org/"); @@ -52,7 +52,6 @@ public static IServiceCollection AddPkgDiffOutboundHttp(this IServiceCollection client.MaxResponseContentBufferSize = RegistryMaxResponseBytes; }); - // pub.dev HTML pages measured at <= 0.3 MB (analyzer/versions), so 8 MiB leaves ample headroom services.AddHttpClient(HttpClientNames.PubDev, client => { client.BaseAddress = new Uri("https://pub.dev/"); @@ -62,7 +61,7 @@ public static IServiceCollection AddPkgDiffOutboundHttp(this IServiceCollection services.AddScoped(); - // Security data providers, called from background jobs only + // Security providers: called from background jobs only AddSecurityApi(services, HttpClientNames.Osv, "https://api.osv.dev/"); AddSecurityApi(services, HttpClientNames.DepsDev, "https://api.deps.dev/"); AddSecurityApi(services, HttpClientNames.Scorecard, "https://api.securityscorecards.dev/"); @@ -74,7 +73,7 @@ public static IServiceCollection AddPkgDiffOutboundHttp(this IServiceCollection client.Timeout = TimeSpan.FromSeconds(5); }); - // Single intentional internal target (host.docker.internal), restricted by ufw per docs/ops/network-hardening.md + // The one intentional internal target (host.docker.internal), restricted by ufw services.AddHttpClient((provider, client) => { var baseUrl = provider.GetRequiredService()["Ollama:BaseUrl"] ?? "http://localhost:11434"; diff --git a/src/backend/PkgDiff.API/Extensions/SecretsServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/SecretsServiceCollectionExtensions.cs index 232bf22..4834022 100644 --- a/src/backend/PkgDiff.API/Extensions/SecretsServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/SecretsServiceCollectionExtensions.cs @@ -12,8 +12,7 @@ public static class SecretsServiceCollectionExtensions public static IServiceCollection AddPkgDiffSecrets(this IServiceCollection services) { - // The key is validated when the protector is first resolved, never during registration, - // so the EF design-time host keeps working without SETTINGS_ENCRYPTION_KEY. + // Key validated on first resolve so EF design-time tooling works without it services.AddSingleton(sp => new AesGcmSecretProtector(ResolveSettingsKey(sp, warnOnDevelopmentFallback: true))); services.AddScoped(); @@ -21,8 +20,7 @@ public static IServiceCollection AddPkgDiffSecrets(this IServiceCollection servi return services; } - // Same key policy as the settings: required outside Development, derived dev-only key in Development. - // Resolved lazily when the key ring is first loaded, never during registration. + // Same key policy as the settings; resolved lazily for EF design-time tooling public static IDataProtectionBuilder ProtectKeysWithSettingsEncryptionKey(this IDataProtectionBuilder builder) { builder.Services.AddSingleton(sp => new KeyRingEncryptionKey(ResolveSettingsKey(sp, warnOnDevelopmentFallback: false))); diff --git a/src/backend/PkgDiff.API/Extensions/ServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/ServiceCollectionExtensions.cs index 76aaed0..aa15f96 100644 --- a/src/backend/PkgDiff.API/Extensions/ServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/ServiceCollectionExtensions.cs @@ -11,18 +11,14 @@ public static class ServiceCollectionExtensions { public static IServiceCollection AddBusinessServices(this IServiceCollection services) { - // SSRF-guarded defaults for every HttpClient, the named clients and the Ollama exception services.AddPkgDiffOutboundHttp(); - // Secret settings protector (key from Security:SettingsEncryptionKey) and decrypting settings reader services.AddPkgDiffSecrets(); - // Rate Limiting (Singleton - global state) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - // Core Registry Services services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -38,30 +34,25 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser client.DefaultRequestHeaders.Add("User-Agent", "PkgDiff/1.0"); }); - // Scrape Log Service services.AddScoped(); - // Scraping Strategies services.AddScoped(); services.AddScoped(); services.AddScoped(); - // One client per registry, found by name through the factory services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); - // Factory & Cache Services services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); - // Analysis Services services.AddSingleton(); services.AddScoped(); // Concrete class for Hangfire - services.AddScoped(); // Interface + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -84,7 +75,7 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser services.AddScoped(); services.AddScoped(); - // AutoPilot & LLM (the Ollama typed client is registered by AddPkgDiffOutboundHttp) + // The Ollama typed client is registered by AddPkgDiffOutboundHttp services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/backend/PkgDiff.API/Extensions/StartupTasks.cs b/src/backend/PkgDiff.API/Extensions/StartupTasks.cs index 5fdcfdb..cc16a0b 100644 --- a/src/backend/PkgDiff.API/Extensions/StartupTasks.cs +++ b/src/backend/PkgDiff.API/Extensions/StartupTasks.cs @@ -9,12 +9,10 @@ namespace PkgDiff.API.Extensions; -// What the host does once before it serves traffic public static class StartupTasks { private const string DefaultAutoPilotCron = "0 8,14,20 * * *"; - // The refresh job is what finds new versions for Live Updates; without the setting it still runs, every six hours private const string DefaultCacheRefreshCron = "0 */6 * * *"; // A database that cannot be migrated stops the application @@ -35,7 +33,7 @@ public static async Task MigrateDatabaseAsync(this WebApplication app) } } - // Logs GitHub configuration and token validity; never any token characters + // Never logs any token characters public static async Task LogGitHubStatusAsync(this WebApplication app) { using var scope = app.Services.CreateScope(); @@ -53,7 +51,7 @@ await GitHubStartupProbe.LogStatusAsync( } } - // Any failure (missing ADMIN_PASSWORD, Identity error) stops startup + // Any failure stops startup public static async Task SeedDatabaseAsync(this WebApplication app) { using var scope = app.Services.CreateScope(); @@ -67,13 +65,11 @@ public static async Task ScheduleRecurringJobsAsync(this WebApplication app) job => job.ExecuteAsync(), app.Configuration["BackgroundJobs:CacheRefreshCron"] ?? DefaultCacheRefreshCron); - // Security data of packages people looked at, refreshed in the background before it is served stale RecurringJob.AddOrUpdate( "security-refresh", job => job.RefreshExpiredAsync(), "15 */6 * * *"); - // Dependency audit reports are kept for 30 days RecurringJob.AddOrUpdate( "dependency-audit-cleanup", job => job.DeleteExpiredAsync(), diff --git a/src/backend/PkgDiff.API/Filters/HangfireAuthorizationFilter.cs b/src/backend/PkgDiff.API/Filters/HangfireAuthorizationFilter.cs index cfe6fd2..186ddba 100644 --- a/src/backend/PkgDiff.API/Filters/HangfireAuthorizationFilter.cs +++ b/src/backend/PkgDiff.API/Filters/HangfireAuthorizationFilter.cs @@ -3,7 +3,7 @@ namespace PkgDiff.API.Filters; -// Re-checks against the database on every dashboard request that the cookie or bearer user is still an active Admin. +// Re-checks the database on every request that the user is still an active Admin public class HangfireAuthorizationFilter : IDashboardAsyncAuthorizationFilter { public Task AuthorizeAsync(DashboardContext context) diff --git a/src/backend/PkgDiff.API/Filters/ValidatePackageRouteAttribute.cs b/src/backend/PkgDiff.API/Filters/ValidatePackageRouteAttribute.cs index 73e5132..25a13f7 100644 --- a/src/backend/PkgDiff.API/Filters/ValidatePackageRouteAttribute.cs +++ b/src/backend/PkgDiff.API/Filters/ValidatePackageRouteAttribute.cs @@ -79,7 +79,6 @@ private static async Task IsRegistryDisabledAsync(ActionExecutingContext c var services = context.HttpContext.RequestServices; var unitOfWork = services.GetRequiredService(); - // pub and pub.dev are aliases for the same registry row var keys = RegistryNames.SpellingsOf(registry); try diff --git a/src/backend/PkgDiff.API/Jobs/DatabaseCleanupJob.cs b/src/backend/PkgDiff.API/Jobs/DatabaseCleanupJob.cs index 6b12289..06180c8 100644 --- a/src/backend/PkgDiff.API/Jobs/DatabaseCleanupJob.cs +++ b/src/backend/PkgDiff.API/Jobs/DatabaseCleanupJob.cs @@ -3,7 +3,6 @@ namespace PkgDiff.API.Jobs; -// Daily Hangfire job: the log tables would otherwise grow without end public class DatabaseCleanupJob { private readonly ApplicationDbContext _dbContext; @@ -15,17 +14,14 @@ public DatabaseCleanupJob(ApplicationDbContext dbContext) public async Task CleanupOldRequestLogsAsync() { - // RequestLogs older than 30 days await _dbContext.Database.ExecuteSqlRawAsync( "DELETE FROM \"RequestLogs\" WHERE \"CreatedAt\" < {0}", DateTime.UtcNow.AddDays(-30)); - // ScrapeLogs older than 30 days await _dbContext.Database.ExecuteSqlRawAsync( "DELETE FROM \"ScrapeLogs\" WHERE \"CreatedAt\" < {0}", DateTime.UtcNow.AddDays(-30)); - // SearchHistories older than 90 days await _dbContext.Database.ExecuteSqlRawAsync( "DELETE FROM \"SearchHistories\" WHERE \"SearchedAt\" < {0}", DateTime.UtcNow.AddDays(-90)); diff --git a/src/backend/PkgDiff.API/Middleware/GlobalExceptionHandler.cs b/src/backend/PkgDiff.API/Middleware/GlobalExceptionHandler.cs index 9d14d3f..fb6e4ce 100644 --- a/src/backend/PkgDiff.API/Middleware/GlobalExceptionHandler.cs +++ b/src/backend/PkgDiff.API/Middleware/GlobalExceptionHandler.cs @@ -4,8 +4,7 @@ namespace PkgDiff.API.Middleware; -// Last line of defence for an exception no action caught: a status code that fits, the detail in -// the log only, and a ProblemDetails body that also carries the "error" field the SPA reads +// Detail goes to the log only; the ProblemDetails body also carries the SPA's "error" field public sealed class GlobalExceptionHandler : IExceptionHandler { private const int ClientClosedRequest = 499; diff --git a/src/backend/PkgDiff.API/Middleware/HangfireTicketMiddleware.cs b/src/backend/PkgDiff.API/Middleware/HangfireTicketMiddleware.cs index 2b5af2b..07e2457 100644 --- a/src/backend/PkgDiff.API/Middleware/HangfireTicketMiddleware.cs +++ b/src/backend/PkgDiff.API/Middleware/HangfireTicketMiddleware.cs @@ -2,8 +2,7 @@ namespace PkgDiff.API.Middleware; -// Exchanges a one-time dashboard ticket for the /hangfire cookie. This cannot live in the Hangfire -// authorization filter because Hangfire overwrites the response status when a filter denies access. +// Not in the authorization filter: Hangfire overwrites the response status when one denies public class HangfireTicketMiddleware { private readonly RequestDelegate _next; diff --git a/src/backend/PkgDiff.API/Middleware/MetricsAccessMiddleware.cs b/src/backend/PkgDiff.API/Middleware/MetricsAccessMiddleware.cs index dfbc85b..888e3b7 100644 --- a/src/backend/PkgDiff.API/Middleware/MetricsAccessMiddleware.cs +++ b/src/backend/PkgDiff.API/Middleware/MetricsAccessMiddleware.cs @@ -33,9 +33,7 @@ public async Task InvokeAsync(HttpContext context) await _next(context); } - // ForwardedHeadersMiddleware records the original peer only when it applied X-Forwarded-For, so - // that header's presence means the request came through the proxy chain from a visitor. The bridge gateway is - // where host-originated traffic (docker-proxy) appears from, so it is not an internal container. + // X-Original-For means a proxied visitor; the bridge gateway carries host-originated traffic private bool IsInternalScrape(HttpContext context) { var peer = context.Connection.RemoteIpAddress; diff --git a/src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs b/src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs index eaf04ac..c733a95 100644 --- a/src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs +++ b/src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs @@ -27,8 +27,7 @@ public async Task InvokeAsync(HttpContext context) var ip = context.GetClientIpAddress(); var path = context.Request.Path; - // Only a signed-in caller gets the larger admin budget: the limiter runs before authentication, - // so an anonymous request to an admin address is counted like any other + // Runs before authentication: only callers with credentials get the admin budget if (IsAdminPath(path) && HasCredentials(context)) { if (await IsOverLimitAsync("rl:admin", ip, _options.Admin)) @@ -42,7 +41,7 @@ public async Task InvokeAsync(HttpContext context) await ReturnRateLimitError(context, "Too many requests. Please slow down."); return; } - // Exports have their own counters: the general ones also count the API calls of the page the export is started from + // Separate counters: the page starting an export already spends the general budget else if (IsPdfExport(path) && await IsOverLimitAsync("rl:pdf", ip, _options.PdfExport)) { await ReturnRateLimitError(context, "Too many requests for PDF exports. Please slow down."); @@ -66,13 +65,12 @@ private static bool IsAdminPath(PathString path) => path.StartsWithSegments("/api/admin", StringComparison.OrdinalIgnoreCase) || path.StartsWithSegments("/hangfire", StringComparison.OrdinalIgnoreCase); - // Both PDF exports: /api/Package/{registry}/{id}/export/pdf and /api/Package/export/diff/pdf private static bool IsPdfExport(PathString path) => path.Value is { } value && value.Contains("/export/", StringComparison.OrdinalIgnoreCase) && value.EndsWith("/pdf", StringComparison.OrdinalIgnoreCase); - // Exactly this address: /api/audit/parse reads a file and starts nothing + // Exact path match, so /api/audit/parse does not count as an audit start private static bool IsPost(HttpContext context, string path) => HttpMethods.IsPost(context.Request.Method) && string.Equals(context.Request.Path.Value?.TrimEnd('/'), path, StringComparison.OrdinalIgnoreCase); diff --git a/src/backend/PkgDiff.API/Middleware/RequestLoggingMiddleware.cs b/src/backend/PkgDiff.API/Middleware/RequestLoggingMiddleware.cs index 0cc0670..6a52088 100644 --- a/src/backend/PkgDiff.API/Middleware/RequestLoggingMiddleware.cs +++ b/src/backend/PkgDiff.API/Middleware/RequestLoggingMiddleware.cs @@ -9,16 +9,13 @@ public class RequestLoggingMiddleware { private readonly RequestDelegate _next; - // Endpoint prefixes that belong to internal/system traffic — never logged. - // Skipping is path-only: behind the proxy every peer is a 172.* address, so IP or - // user-agent based skips would silently drop real visitor traffic. + // Path-only: behind the proxy every peer is 172.*, so IP-based skips would drop visitors private static readonly string[] _skipEndpoints = [ "/health", "/metrics", "/swagger", "/hangfire", "/analysisHub", "/favicon", "/_framework", ]; - // File extensions that are static assets — never logged private static readonly string[] _skipExtensions = [ ".js", ".css", ".ico", ".png", ".jpg", ".jpeg", diff --git a/src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs b/src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs index 1150886..f8899a6 100644 --- a/src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs +++ b/src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs @@ -6,16 +6,13 @@ public sealed class InboundRateLimitOptions public RateLimitTier General { get; set; } = new() { PerMinute = 60, PerFiveMinutes = 200 }; - // Counted on its own, on top of the general tier: an export is far more expensive than an API call + // On top of the general tier: an export costs far more than an API call public RateLimitTier PdfExport { get; set; } = new() { PerMinute = 5, PerFiveMinutes = 15 }; - // The admin panel and the Hangfire dashboard poll, so signed-in work needs more room than a visitor public RateLimitTier Admin { get; set; } = new() { PerMinute = 240, PerFiveMinutes = 1200 }; - // Sign-in attempts per address: tight, so guessing passwords is slow public RateLimitTier Login { get; set; } = new() { PerMinute = 5, PerFiveMinutes = 15 }; - // Each audit checks up to the package limit against OSV: a few per address are plenty public RateLimitTier AuditStart { get; set; } = new() { PerMinute = 3, PerFiveMinutes = 6 }; public bool IsValid() => diff --git a/src/backend/PkgDiff.API/Options/TrustedNetworks.cs b/src/backend/PkgDiff.API/Options/TrustedNetworks.cs index 77ce119..6b9fdae 100644 --- a/src/backend/PkgDiff.API/Options/TrustedNetworks.cs +++ b/src/backend/PkgDiff.API/Options/TrustedNetworks.cs @@ -21,8 +21,7 @@ public bool Contains(IPAddress address) n.BaseAddress.AddressFamily == normalized.AddressFamily && n.Contains(normalized)); } - // The Docker bridge gateway (e.g. 172.28.0.1) is where host-originated traffic appears from, - // so it must never count as an internal container peer. + // The bridge gateway carries host-originated traffic, so it is never an internal peer public bool IsGateway(IPAddress address) { var normalized = Normalize(address); diff --git a/src/backend/PkgDiff.API/Program.cs b/src/backend/PkgDiff.API/Program.cs index 0bdff27..e5e0fcc 100644 --- a/src/backend/PkgDiff.API/Program.cs +++ b/src/backend/PkgDiff.API/Program.cs @@ -18,7 +18,6 @@ builder.Services.AddPkgDiffPersistence(builder.Configuration); -// JWT authentication (validated JwtSettings) and permission policies builder.Services.AddPkgDiffAuthentication(builder.Configuration); builder.Services.AddPkgDiffAuthorization(); @@ -28,7 +27,6 @@ builder.Services.AddPkgDiffCaching(builder.Configuration); builder.Services.AddPkgDiffDiagnostics(); -// Forwarded headers (trusted proxy networks from ForwardedHeaders:TrustedNetworks) builder.Services.AddPkgDiffForwardedHeaders(builder.Configuration); builder.Services.AddPkgDiffCors(); @@ -40,8 +38,7 @@ await app.MigrateDatabaseAsync(); await app.LogGitHubStatusAsync(); -// The HTTP request pipeline; the order below matters. -// Must run first so every later middleware sees the real client IP and scheme. +// Pipeline order matters; forwarded headers first so later middleware sees the real client IP app.UseForwardedHeaders(); app.UseExceptionHandler(); @@ -55,7 +52,7 @@ } else { - // No HTTPS redirect here: TLS ends at the host nginx, which redirects HTTP itself; the container has no HTTPS port + // No HTTPS redirect: TLS ends at the host nginx app.UseCors(HostingServiceCollectionExtensions.ProductionCorsPolicy); app.UseHsts(); } @@ -63,20 +60,18 @@ app.UseDefaultFiles(); app.UseStaticFiles(); -// Security middleware (must be before authentication) +// Must run before authentication app.UseMiddleware(); app.UseMiddleware(); app.UseMiddleware(); -app.UseAuthentication(); // Must be before UseAuthorization +app.UseAuthentication(); app.UseAuthorization(); -// Prometheus metrics: internal scrape (trusted network, not the bridge gateway) or Admin only app.UseMiddleware(); -app.UseMetricServer(); // Exposes /metrics -app.UseHttpMetrics(); // Captures HTTP metrics +app.UseMetricServer(); +app.UseHttpMetrics(); -// Hangfire dashboard, active Admins only: /hangfire?ticket= becomes a scoped cookie, re-checked on every request app.UseMiddleware(); app.UseHangfireDashboard("/hangfire", new DashboardOptions { @@ -101,7 +96,6 @@ await app.SeedDatabaseAsync(); -// After all middleware and initialization await app.ScheduleRecurringJobsAsync(); app.Run(); diff --git a/src/backend/PkgDiff.API/Services/GitHubStartupProbe.cs b/src/backend/PkgDiff.API/Services/GitHubStartupProbe.cs index 0902c4f..72f0e78 100644 --- a/src/backend/PkgDiff.API/Services/GitHubStartupProbe.cs +++ b/src/backend/PkgDiff.API/Services/GitHubStartupProbe.cs @@ -11,7 +11,7 @@ public static class GitHubStartupProbe private const int UnauthenticatedHourlyLimit = 60; private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5); - // Never log any part of the token: prefixes/suffixes narrow brute force and identify the token in leaked logs. + // Never log any part of the token: even fragments identify it in leaked logs public static async Task LogStatusAsync( IConfiguration configuration, IHttpClientFactory httpClientFactory, diff --git a/src/backend/PkgDiff.API/Services/HangfireDashboardAccess.cs b/src/backend/PkgDiff.API/Services/HangfireDashboardAccess.cs index cf4b02b..a2d691e 100644 --- a/src/backend/PkgDiff.API/Services/HangfireDashboardAccess.cs +++ b/src/backend/PkgDiff.API/Services/HangfireDashboardAccess.cs @@ -37,7 +37,7 @@ public string CreateTicket(string userId) => public string CreateTicket(string userId, DateTimeOffset expiresAt) { - // The random nonce identifies the ticket for the single-use marker independently of its textual encoding. + // The nonce keys the single-use marker independently of the ticket's encoding var nonce = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)); return _ticketProtector.Protect(nonce + NonceSeparator + userId, expiresAt); } @@ -64,8 +64,7 @@ public string CreateTicket(string userId, DateTimeOffset expiresAt) var nonce = payload[..separator]; var userId = payload[(separator + 1)..]; - // IDistributedCache has no atomic check-and-set: two redemptions racing in the same instant are - // bounded by the 60 s ticket lifetime and the per-request Admin re-check. + // No atomic check-and-set: a same-instant race is bounded by the 60 s ticket lifetime var markerKey = "hangfire-ticket-used:" + nonce; if (await _cache.GetAsync(markerKey) is not null) return null; @@ -95,8 +94,7 @@ public string CreateCookieValue(string userId) => public async Task IsAuthorizedAsync(HttpContext context, IAdminRoleChecker checker) { - // A JWT role claim lives for up to 24 h and cannot be revoked, so the bearer path gets the same - // "still an active Admin" database check as the cookie path + // JWT role claims cannot be revoked, so the bearer path also gets the database Admin check if (context.User.IsInRole("Admin")) { var bearerUserId = context.User.FindFirstValue(ClaimTypes.NameIdentifier) ?? context.User.FindFirstValue("sub"); diff --git a/src/backend/PkgDiff.API/Services/IRateLimitCounterStore.cs b/src/backend/PkgDiff.API/Services/IRateLimitCounterStore.cs index 5861444..d307a9c 100644 --- a/src/backend/PkgDiff.API/Services/IRateLimitCounterStore.cs +++ b/src/backend/PkgDiff.API/Services/IRateLimitCounterStore.cs @@ -2,9 +2,6 @@ namespace PkgDiff.API.Services; public interface IRateLimitCounterStore { - /// - /// Atomically increments the counter for the given key and returns the new value. - /// Sets the TTL on first increment. - /// + // Atomic; the TTL is set on the first increment only Task IncrementAsync(string key, TimeSpan ttl); } diff --git a/src/backend/PkgDiff.API/Services/MemoryRateLimitCounterStore.cs b/src/backend/PkgDiff.API/Services/MemoryRateLimitCounterStore.cs index 7495698..c6ba3f1 100644 --- a/src/backend/PkgDiff.API/Services/MemoryRateLimitCounterStore.cs +++ b/src/backend/PkgDiff.API/Services/MemoryRateLimitCounterStore.cs @@ -2,8 +2,7 @@ namespace PkgDiff.API.Services; -// Fixed windows: the first request of a key starts its window, and the window ends when its time is up, -// however many requests arrive in between +// Fixed windows, each starting at the first request of its key public class MemoryRateLimitCounterStore : IRateLimitCounterStore { private static readonly TimeSpan SweepInterval = TimeSpan.FromMinutes(1); @@ -54,7 +53,6 @@ public Task IncrementAsync(string key, TimeSpan ttl) } } - // Keeps the dictionary from growing with every address ever seen private void Sweep(DateTime now) { var due = Interlocked.Read(ref _nextSweepTicks); diff --git a/src/backend/PkgDiff.API/Services/ReadinessHealthChecks.cs b/src/backend/PkgDiff.API/Services/ReadinessHealthChecks.cs index 0db04e3..13fd89b 100644 --- a/src/backend/PkgDiff.API/Services/ReadinessHealthChecks.cs +++ b/src/backend/PkgDiff.API/Services/ReadinessHealthChecks.cs @@ -54,7 +54,7 @@ public static class HealthResponses { public const string ReadyTag = "ready"; - // Names and states only: a health endpoint is unauthenticated, so it never carries exception text + // Unauthenticated endpoint: names and states only, never exception text public static Task WriteAsync(HttpContext context, HealthReport report) { context.Response.ContentType = "application/json"; diff --git a/src/backend/PkgDiff.API/Services/RedisRateLimitCounterStore.cs b/src/backend/PkgDiff.API/Services/RedisRateLimitCounterStore.cs index be34e7f..b1f678f 100644 --- a/src/backend/PkgDiff.API/Services/RedisRateLimitCounterStore.cs +++ b/src/backend/PkgDiff.API/Services/RedisRateLimitCounterStore.cs @@ -4,8 +4,7 @@ namespace PkgDiff.API.Services; public class RedisRateLimitCounterStore : IRateLimitCounterStore { - // One script, so a counter can never be left without an expiry: INCR followed by a separate - // EXPIRE would block an address for good if the process stopped between the two commands + // One script, so a crash between INCR and EXPIRE cannot leave a counter without expiry private const string IncrementScript = @" local count = redis.call('INCR', KEYS[1]) if count == 1 then diff --git a/src/backend/PkgDiff.API/Services/RequestLogBatchService.cs b/src/backend/PkgDiff.API/Services/RequestLogBatchService.cs index e7d8cc4..f3cc10f 100644 --- a/src/backend/PkgDiff.API/Services/RequestLogBatchService.cs +++ b/src/backend/PkgDiff.API/Services/RequestLogBatchService.cs @@ -32,7 +32,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { - // Wait for the next item or flush interval using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); cts.CancelAfter(_flushInterval); @@ -57,7 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogError(ex, "Error processing log batch"); - await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); // Backoff + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } } diff --git a/src/backend/PkgDiff.Application/Diagnostics/PkgDiffTelemetry.cs b/src/backend/PkgDiff.Application/Diagnostics/PkgDiffTelemetry.cs index c9abc32..6d04270 100644 --- a/src/backend/PkgDiff.Application/Diagnostics/PkgDiffTelemetry.cs +++ b/src/backend/PkgDiff.Application/Diagnostics/PkgDiffTelemetry.cs @@ -4,10 +4,8 @@ namespace PkgDiff.Application.Diagnostics; -// The application's own metrics and traces, through the .NET APIs that OpenTelemetry collects: -// one Meter and one ActivitySource, both named "PkgDiff". The /metrics endpoint publishes the meter -// as it is; an OpenTelemetry SDK added to the host only has to subscribe to this name. -// Tags are kept to small fixed sets. A package id goes on a trace, never on a metric. +// The app's own metrics and traces: one Meter and one ActivitySource, both named "PkgDiff". +// Tags stay small fixed sets: a package id goes on a trace, never on a metric. public static class PkgDiffTelemetry { public const string Name = "PkgDiff"; diff --git a/src/backend/PkgDiff.Application/Interfaces/IAutoPilotService.cs b/src/backend/PkgDiff.Application/Interfaces/IAutoPilotService.cs index 031ce00..ef0ae80 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IAutoPilotService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IAutoPilotService.cs @@ -4,9 +4,7 @@ namespace PkgDiff.Application.Interfaces; public interface IAutoPilotService { - // Hangfire reads filters from the interface method it enqueues. A failed draft is not - // retried: the next scheduled run tries again, and retries of a slow CPU-bound LLM call - // used to pile up behind each other and block the workers. + // Hangfire reads filters from the interface. No retries: the next scheduled run tries again. [AutomaticRetry(Attempts = 0)] [DisableConcurrentExecution(timeoutInSeconds: 600)] Task RunAutoPilotJobAsync(); diff --git a/src/backend/PkgDiff.Application/Interfaces/IDependencyAuditService.cs b/src/backend/PkgDiff.Application/Interfaces/IDependencyAuditService.cs index b90879d..3ee4d7c 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IDependencyAuditService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IDependencyAuditService.cs @@ -4,21 +4,17 @@ namespace PkgDiff.Application.Interfaces; public interface IDependencyAuditService { - // Stores the request and queues the background job; returns the id of the report Task StartAsync(StartAuditRequest request); - // null when there is no such report or it has expired Task GetAsync(Guid id); - // Audits waiting or running; the start endpoint refuses new ones above a limit Task CountActiveAsync(); - // Background job, on its own queue: a burst of audits must not hold the workers scrapes need + // Own queue: a burst of audits must not hold the workers scrapes need [Hangfire.Queue(AuditQueue)] Task RunAsync(Guid id); const string AuditQueue = "audits"; - // Recurring job Task DeleteExpiredAsync(); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IExportService.cs b/src/backend/PkgDiff.Application/Interfaces/IExportService.cs index fe5506e..e57aea8 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IExportService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IExportService.cs @@ -4,13 +4,7 @@ namespace PkgDiff.Application.Interfaces; public interface IExportService { - /// - /// Export all release notes for a package to PDF format - /// Task ExportToPdfAsync(PackageMetadata metadata); - /// - /// Export diff analysis results to PDF format - /// Task ExportDiffToPdfAsync(AnalysisResult diff); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IGitHubRateLimitHandler.cs b/src/backend/PkgDiff.Application/Interfaces/IGitHubRateLimitHandler.cs index be8183c..d7c25b8 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IGitHubRateLimitHandler.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IGitHubRateLimitHandler.cs @@ -1,26 +1,11 @@ namespace PkgDiff.Application.Interfaces; -/// -/// Service for tracking and managing GitHub API rate limits based on response headers -/// public interface IGitHubRateLimitHandler { - /// - /// Counts one request against the budget, or throws - /// when the budget is spent until the rate limit resets - /// - /// True if request can proceed + // Throws GitHubRateLimitedException while the budget is spent until the reset Task CanMakeRequestAsync(); - /// - /// Updates rate limit state from GitHub API response headers - /// - /// HTTP response containing rate limit headers Task UpdateFromHeadersAsync(HttpResponseMessage response); - /// - /// Gets the current rate limit status - /// - /// Tuple of (remaining, limit, reset time) Task<(int Remaining, int Limit, DateTime? ResetTime)> GetRateLimitStatusAsync(); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IGitHubScraperService.cs b/src/backend/PkgDiff.Application/Interfaces/IGitHubScraperService.cs index b08a26f..db87464 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IGitHubScraperService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IGitHubScraperService.cs @@ -2,23 +2,13 @@ namespace PkgDiff.Application.Interfaces; public interface IGitHubScraperService { - /// - /// Extracts the GitHub repository URL from a NuGet package page - /// Task ExtractGitHubRepositoryUrlAsync(string packageId); - /// - /// Gets all releases from a GitHub repository - /// - /// Dictionary with version as key and release notes as value Task> GetAllReleasesAsync(string githubRepoUrl); - /// - /// Gets release notes for a specific version from GitHub Releases - /// Task GetReleaseNotesForVersionAsync(string githubRepoUrl, string version); - // With the package name a monorepo's tags ("socket.io@4.8.3", "@scope/core@1.2.0") can be told apart + // The package name tells a monorepo's per-package tags apart Task GetReleaseNotesForVersionAsync(string githubRepoUrl, string version, string? packageId) => GetReleaseNotesForVersionAsync(githubRepoUrl, version); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IJobScheduler.cs b/src/backend/PkgDiff.Application/Interfaces/IJobScheduler.cs index 943cb58..cf1debc 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IJobScheduler.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IJobScheduler.cs @@ -2,8 +2,6 @@ namespace PkgDiff.Application.Interfaces; -// Queues work for the background workers. The caller names the service and the call; which job -// system stores and runs it is not its concern. public interface IJobScheduler { string Enqueue(Expression> job); diff --git a/src/backend/PkgDiff.Application/Interfaces/IPackageTrustService.cs b/src/backend/PkgDiff.Application/Interfaces/IPackageTrustService.cs index 628ca0d..3c7a6f2 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IPackageTrustService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IPackageTrustService.cs @@ -7,9 +7,7 @@ public interface IPackageTrustService // Reads the stored data only and queues a refresh when it is missing or old; never calls a provider Task GetTrustAsync(string registry, string packageId); - // Background job: asks the providers and stores the result Task RefreshAsync(string registry, string packageId); - // Recurring job: refreshes the stored packages whose data has expired Task RefreshExpiredAsync(); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IRateLimiterService.cs b/src/backend/PkgDiff.Application/Interfaces/IRateLimiterService.cs index 62f7464..50aea9a 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IRateLimiterService.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IRateLimiterService.cs @@ -1,24 +1,10 @@ namespace PkgDiff.Application.Interfaces; -/// -/// Service for managing rate limits for external API calls (e.g., NuGet.org web scraping) -/// public interface IRateLimiterService { - /// - /// Waits if necessary to respect rate limits, then executes the action - /// - /// Rate limit key (e.g., "nuget-scraping") - /// Action to execute Task ExecuteAsync(string key, Func> action); - /// - /// Checks if the rate limit allows execution without waiting - /// bool CanExecute(string key); - /// - /// Gets the time until the next available slot - /// TimeSpan GetTimeUntilNextSlot(string key); } diff --git a/src/backend/PkgDiff.Application/Interfaces/IRegistryClient.cs b/src/backend/PkgDiff.Application/Interfaces/IRegistryClient.cs index 9ba5c6c..7aa2ce0 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IRegistryClient.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IRegistryClient.cs @@ -3,14 +3,11 @@ namespace PkgDiff.Application.Interfaces; -// Everything that differs from one registry to the next. Each registry has exactly one client, -// found by name through IRegistryServiceFactory. public interface IRegistryClient { - // One of RegistryNames string Id { get; } - // The release-notes source recorded for a version whose notes come from the registry itself + // Source recorded for notes that come from the registry itself string SourceLabel { get; } VersionScheme VersionScheme { get; } @@ -21,7 +18,7 @@ public interface IRegistryClient // False when the notes of a package can only come from its repository bool PublishesReleaseNotes { get; } - // True when the metadata of a package names its folder in the repository (PackageMetadata.SourceDirectory) + // True when package metadata names its folder in the repository bool KnowsSourceDirectory => false; Task PackageExistsAsync(string packageId, bool includePrerelease); @@ -33,7 +30,6 @@ public interface IRegistryClient Task GetMetadataAsync(string packageId); - // The notes the registry itself publishes for a version; null when it publishes none Task GetOwnReleaseNotesAsync(string packageId, string version); IReleaseNotesScrapingStrategy GetScrapingStrategy(string? githubRepoUrl); diff --git a/src/backend/PkgDiff.Application/Interfaces/IReleaseNotesUrlResolver.cs b/src/backend/PkgDiff.Application/Interfaces/IReleaseNotesUrlResolver.cs index bafd9ad..de0f869 100644 --- a/src/backend/PkgDiff.Application/Interfaces/IReleaseNotesUrlResolver.cs +++ b/src/backend/PkgDiff.Application/Interfaces/IReleaseNotesUrlResolver.cs @@ -1,16 +1,8 @@ namespace PkgDiff.Application.Interfaces; -/// -/// Detects URLs in release notes text that point to actual release notes content, -/// fetches and returns the resolved content. Returns original text if no URL resolution needed. -/// public interface IReleaseNotesUrlResolver { - /// - /// Inspects release notes text. If the text is URL-dominated (contains URLs but little - /// substantive content), extracts URLs, fetches their content, and returns the resolved - /// release notes. Otherwise returns the original text unchanged. - /// + // Fetches the linked content when the notes are mostly URLs; otherwise returns them unchanged Task ResolveAsync( string rawReleaseNotes, string packageId, @@ -18,10 +10,6 @@ Task ResolveAsync( string? githubRepoUrl); } -/// The resolved (or original) release notes text -/// True if URLs were detected and content was fetched from them -/// Source description if resolved -/// GitHub repo URL discovered during resolution, if any public record UrlResolverResult( string ReleaseNotes, bool WasResolved, diff --git a/src/backend/PkgDiff.Application/Jobs/BlogSocialPostJob.cs b/src/backend/PkgDiff.Application/Jobs/BlogSocialPostJob.cs index 24b4d1d..2938d90 100644 --- a/src/backend/PkgDiff.Application/Jobs/BlogSocialPostJob.cs +++ b/src/backend/PkgDiff.Application/Jobs/BlogSocialPostJob.cs @@ -61,8 +61,7 @@ public async Task ExecuteAsync(int blogPostId) if (previousPosts.Any(p => p.Platform == platform && p.Status == SocialMediaPostStatus.Success)) continue; - // A Pending row means an earlier attempt reached the platform but its outcome was never recorded; - // posting again could publish a duplicate, so it is left for an admin to check + // Pending means an earlier attempt's outcome is unknown; reposting could publish a duplicate if (previousPosts.Any(p => p.Platform == platform && p.Status == SocialMediaPostStatus.Pending)) { _logger.LogWarning( diff --git a/src/backend/PkgDiff.Application/Jobs/PackageAnalysisJob.cs b/src/backend/PkgDiff.Application/Jobs/PackageAnalysisJob.cs index d2a1e50..a50328e 100644 --- a/src/backend/PkgDiff.Application/Jobs/PackageAnalysisJob.cs +++ b/src/backend/PkgDiff.Application/Jobs/PackageAnalysisJob.cs @@ -29,10 +29,7 @@ public async Task ExecuteAsync() try { - // Get all cached packages with their registry info - // Only what the refresh request needs, not the cached version lists - // Only packages someone looked at in the last 30 days (before views were tracked: cached then). - // Anyone can add a package by opening it; one nobody visits is refreshed when it is opened again. + // Only packages viewed in the last 30 days; an unvisited one refreshes when it is opened again var cutoff = DateTime.UtcNow - RefreshWindow; var packages = await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query() .Where(p => (p.LastViewedAt ?? p.CachedAt) > cutoff) @@ -46,13 +43,12 @@ public async Task ExecuteAsync() { PackageId = package.PackageId, Registry = package.Registry, - CurrentVersion = string.Empty, // Will fetch latest - TargetVersion = string.Empty, // Will fetch latest + CurrentVersion = string.Empty, + TargetVersion = string.Empty, IncludePrerelease = false }; - // One job per package, run in parallel by the Hangfire workers; a package that cannot - // be compared is skipped there instead of failing + // One job per package; one that cannot be compared is skipped there instead of failing _jobs.Enqueue(x => x.RefreshAsync(request)); } diff --git a/src/backend/PkgDiff.Application/Jobs/PackageRefreshJob.cs b/src/backend/PkgDiff.Application/Jobs/PackageRefreshJob.cs index 8004b6d..8fb998f 100644 --- a/src/backend/PkgDiff.Application/Jobs/PackageRefreshJob.cs +++ b/src/backend/PkgDiff.Application/Jobs/PackageRefreshJob.cs @@ -4,10 +4,7 @@ namespace PkgDiff.Application.Jobs; -// Refreshes one cached package for the recurring dispatcher (PackageAnalysisJob). -// A package that cannot be compared, because its registry lists a single version or no longer -// knows it, was cached when someone opened its page. That is not a failure of the refresh: every -// later run would fail the same way and fill the failed-job list of the dashboard. +// A package that cannot be compared is skipped, not failed: every later run would fail the same way. public class PackageRefreshJob { private readonly IPackageAnalysisService _analysisService; diff --git a/src/backend/PkgDiff.Application/Security/TrustDiff.cs b/src/backend/PkgDiff.Application/Security/TrustDiff.cs index 8aaeb25..f57f055 100644 --- a/src/backend/PkgDiff.Application/Security/TrustDiff.cs +++ b/src/backend/PkgDiff.Application/Security/TrustDiff.cs @@ -2,8 +2,7 @@ namespace PkgDiff.Application.Security; -// The security side of an upgrade: what the FROM version suffers from that the TO version no longer -// does, and what TO brings that FROM did not have +// What an upgrade fixes (in FROM, gone in TO) and what it introduces (in TO, not in FROM) public static class TrustDiff { public static IReadOnlyList AdvisoriesOf(PackageTrust trust, string version) diff --git a/src/backend/PkgDiff.Application/Services/AdminDashboardService.cs b/src/backend/PkgDiff.Application/Services/AdminDashboardService.cs index c3d7690..57c7f87 100644 --- a/src/backend/PkgDiff.Application/Services/AdminDashboardService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminDashboardService.cs @@ -19,8 +19,6 @@ public AdminDashboardService(IUnitOfWork unitOfWork) private const string SystemCaller = "System"; - // ───── Dashboard ───── - public async Task GetDashboardStatsAsync() { // Background refreshes record "System" as their caller; they are not searches @@ -71,8 +69,6 @@ public async Task GetDashboardStatsAsync() }; } - // ───── Search History ───── - public async Task> GetSearchHistoryAsync(int page = 1, int pageSize = 20) { (page, pageSize) = Paging.Clamp(page, pageSize); diff --git a/src/backend/PkgDiff.Application/Services/AdminJobService.cs b/src/backend/PkgDiff.Application/Services/AdminJobService.cs index 9b60153..a344418 100644 --- a/src/backend/PkgDiff.Application/Services/AdminJobService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminJobService.cs @@ -27,7 +27,6 @@ public Task GetJobsOverviewAsync() LastJobState = j.LastJobState ?? "None" }).ToList(); - // Recent completed jobs — fetch more to capture full batches var succeeded = monitor.SucceededJobs(0, 50); var failed = monitor.FailedJobs(0, 50); @@ -63,7 +62,6 @@ public Task GetJobsOverviewAsync() }); } - // Group: parent (ExecuteAsync) jobs get children attached var parentJobs = allJobs .Where(j => j.JobName.Contains("ExecuteAsync")) .OrderByDescending(j => j.CreatedAt) @@ -77,7 +75,6 @@ public Task GetJobsOverviewAsync() foreach (var parent in parentJobs) { - // Match children created within 5 seconds of parent's CreatedAt var children = childPool .Where(c => parent.CreatedAt.HasValue && c.CreatedAt.HasValue && Math.Abs((c.CreatedAt.Value - parent.CreatedAt.Value).TotalSeconds) < 5) @@ -92,7 +89,6 @@ public Task GetJobsOverviewAsync() recentJobs.Add(parent); } - // Any unmatched children as standalone entries recentJobs.AddRange(childPool); recentJobs = recentJobs @@ -100,7 +96,6 @@ public Task GetJobsOverviewAsync() .Take(20) .ToList(); - // Active queue jobs (enqueued + processing) var queuedJobs = new List(); var queues = monitor.Queues(); @@ -157,11 +152,9 @@ private static string GetJobDisplayName(Hangfire.Common.Job? job) { if (job?.Args == null || job.Args.Count == 0) return null; - // Try direct cast (Hangfire deserializes to the actual type) if (job.Args[0] is AnalyzeRequest request) return request.PackageId; - // Fallback: parse from serialized JSON string try { var json = job.Args[0]?.ToString(); diff --git a/src/backend/PkgDiff.Application/Services/AdminPackageService.cs b/src/backend/PkgDiff.Application/Services/AdminPackageService.cs index f1b562a..4661310 100644 --- a/src/backend/PkgDiff.Application/Services/AdminPackageService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminPackageService.cs @@ -17,8 +17,6 @@ public AdminPackageService(IUnitOfWork unitOfWork) _unitOfWork = unitOfWork; } - // ───── Packages ───── - public async Task> GetPackagesAsync(int page = 1, int pageSize = 20, string? search = null) { (page, pageSize) = Paging.Clamp(page, pageSize); @@ -48,15 +46,12 @@ public async Task DeletePackageAsync(int id) var entity = await _unitOfWork.CachedPackages.GetByIdAsync(id); if (entity == null) throw new KeyNotFoundException($"Package with id {id} not found"); - // Delete related release notes first await _unitOfWork.CachedReleaseNotes.DeleteWhereAsync(r => r.CachedPackageId == id); await _unitOfWork.CachedPackages.DeleteAsync(entity); await _unitOfWork.SaveChangesAsync(); } - // ───── Registries ───── - public async Task> GetRegistriesAsync() { var registries = await _unitOfWork.Registries.GetAllAsync(); @@ -148,8 +143,6 @@ public async Task ReorderRegistriesAsync(List requests) await _unitOfWork.SaveChangesAsync(); } - // ───── Sitemap ───── - public async Task> GetPackagesForSitemapAsync() { return await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query() diff --git a/src/backend/PkgDiff.Application/Services/AdminRoleService.cs b/src/backend/PkgDiff.Application/Services/AdminRoleService.cs index 845c903..b3544ac 100644 --- a/src/backend/PkgDiff.Application/Services/AdminRoleService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminRoleService.cs @@ -66,8 +66,7 @@ public async Task DeleteRoleAsync(string id, bool callerIsAdmin, IReadOnlyCollec throw new InvalidOperationException("The 'Admin' role is a system role and cannot be deleted."); } - // Deleting a role revokes its permissions from every member: a non-Admin may do that only for - // permissions they hold, as when editing the role + // Deleting a role revokes its permissions, so a non-Admin must hold them, as when editing if (!callerIsAdmin) { var rolePermissions = (await _roleManager.GetClaimsAsync(role)).Where(c => c.Type == "Permission").Select(c => c.Value); @@ -143,8 +142,7 @@ public async Task UpdateRolePermissionsAsync( var currentClaims = await _roleManager.GetClaimsAsync(role); var permissionClaims = currentClaims.Where(c => c.Type == "Permission").ToList(); - // Permission policies pass on the claim alone, so a non-Admin may only grant or revoke what they hold; - // otherwise Roles.Edit would let them hand their own role every permission + // Policies check the claim alone, so a non-Admin may only grant or revoke permissions they hold if (!callerIsAdmin) { var currentPermissions = permissionClaims.Select(c => c.Value).ToHashSet(StringComparer.Ordinal); @@ -156,13 +154,11 @@ public async Task UpdateRolePermissionsAsync( } } - // Remove existing permission claims foreach (var claim in permissionClaims) { await _roleManager.RemoveClaimAsync(role, claim); } - // Add new permission claims foreach (var permission in requestedPermissions) { await _roleManager.AddClaimAsync(role, new System.Security.Claims.Claim("Permission", permission)); diff --git a/src/backend/PkgDiff.Application/Services/AdminSettingsService.cs b/src/backend/PkgDiff.Application/Services/AdminSettingsService.cs index 44a2d14..e45b836 100644 --- a/src/backend/PkgDiff.Application/Services/AdminSettingsService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminSettingsService.cs @@ -82,15 +82,15 @@ public static string GetCronExpressionForFrequency(int dailyFrequency) { return dailyFrequency switch { - 1 => "0 12 * * *", // Once a day at noon - 2 => "0 0,12 * * *", // Every 12 hours - 3 => "0 8,14,20 * * *", // 3 times a day - 4 => "0 0,6,12,18 * * *", // Every 6 hours - 6 => "0 0,4,8,12,16,20 * * *", // Every 4 hours - 8 => "0 0,3,6,9,12,15,18,21 * * *", // Every 3 hours - 12 => "0 0,2,4,6,8,10,12,14,16,18,20,22 * * *", // Every 2 hours - _ when dailyFrequency >= 24 => "0 * * * *", // Every hour - _ => "0 8,14,20 * * *" // Default fallback + 1 => "0 12 * * *", + 2 => "0 0,12 * * *", + 3 => "0 8,14,20 * * *", + 4 => "0 0,6,12,18 * * *", + 6 => "0 0,4,8,12,16,20 * * *", + 8 => "0 0,3,6,9,12,15,18,21 * * *", + 12 => "0 0,2,4,6,8,10,12,14,16,18,20,22 * * *", + _ when dailyFrequency >= 24 => "0 * * * *", + _ => "0 8,14,20 * * *" }; } } diff --git a/src/backend/PkgDiff.Application/Services/AdminUserService.cs b/src/backend/PkgDiff.Application/Services/AdminUserService.cs index 28aa50e..278e944 100644 --- a/src/backend/PkgDiff.Application/Services/AdminUserService.cs +++ b/src/backend/PkgDiff.Application/Services/AdminUserService.cs @@ -307,7 +307,7 @@ public async Task ResetUserPasswordAsync( var user = await _userManager.FindByIdAsync(id); if (user == null) throw new KeyNotFoundException($"User with id {id} not found"); - // Resetting an Admin's password is an account takeover, so it needs the same Admin-only rule as granting Admin. + // Resetting an Admin's password is an account takeover, so it is Admin-only await ThrowIfNonAdminTargetsAdminAsync(user, callerIsAdmin, "Only administrators can reset an administrator's password."); await ThrowIfTargetExceedsCallerAsync(user, callerIsAdmin, callerPermissions, "You cannot reset the password of a user with permissions you do not hold."); diff --git a/src/backend/PkgDiff.Application/Services/AuthService.cs b/src/backend/PkgDiff.Application/Services/AuthService.cs index 1e6f004..a5f6953 100644 --- a/src/backend/PkgDiff.Application/Services/AuthService.cs +++ b/src/backend/PkgDiff.Application/Services/AuthService.cs @@ -38,17 +38,14 @@ public AuthService( return null; } - // No account lockout: with a public user name anyone could keep the admin locked out by failing on - // purpose. Guessing is slowed per address instead (the sign-in rate tier and AuthController). + // No lockout: anyone could lock out a known admin name; guessing is throttled per address var result = await _signInManager.CheckPasswordSignInAsync(user, request.Password, lockoutOnFailure: false); if (!result.Succeeded) return null; - // Update last login user.LastLoginAt = DateTime.UtcNow; await _userManager.UpdateAsync(user); - // Generate JWT token var token = await GenerateJwtToken(user); var roles = await _userManager.GetRolesAsync(user); var permissions = new List(); @@ -153,7 +150,6 @@ private async Task GenerateJwtToken(ApplicationUser user) new Claim(SecurityStampClaim, await _userManager.GetSecurityStampAsync(user) ?? string.Empty) }; - // Add role claims and permissions foreach (var roleName in roles) { claims.Add(new Claim(ClaimTypes.Role, roleName)); diff --git a/src/backend/PkgDiff.Application/Services/AutoPilotOutputCleaner.cs b/src/backend/PkgDiff.Application/Services/AutoPilotOutputCleaner.cs index 7899d1b..1d5082b 100644 --- a/src/backend/PkgDiff.Application/Services/AutoPilotOutputCleaner.cs +++ b/src/backend/PkgDiff.Application/Services/AutoPilotOutputCleaner.cs @@ -23,9 +23,7 @@ public static class AutoPilotOutputCleaner AutoPilotQualityGate.UpgradeNotesHeading }; - // Small models rarely reproduce the exact heading text; these are the variants seen in practice. - // Synonyms only apply to top-level headings (H1/H2), bold lines and bare lines, so an H3 such as - // "### Changes" inside a section is not promoted to a second required section. + // Heading variants small models produce; synonyms apply only to H1/H2, bold and bare lines private static readonly Dictionary HeadingSynonyms = new(StringComparer.Ordinal) { ["summary"] = AutoPilotQualityGate.OverviewHeading, @@ -253,8 +251,7 @@ private static string NormalizeSpacing(string line) return line[..start] + body; } - // Returns the canonical "## ..." heading when the line is a required section in any common form - // (H1-H3, **bold**, bare "Summary:" line, numbered, curly apostrophe), otherwise null + // The canonical "## ..." heading when the line is a required section in any common form, else null private static string? CanonicalizeHeading(string trimmed) { if (trimmed.Length == 0) diff --git a/src/backend/PkgDiff.Application/Services/AutoPilotService.cs b/src/backend/PkgDiff.Application/Services/AutoPilotService.cs index 62bb431..8453197 100644 --- a/src/backend/PkgDiff.Application/Services/AutoPilotService.cs +++ b/src/backend/PkgDiff.Application/Services/AutoPilotService.cs @@ -13,9 +13,7 @@ public class AutoPilotService : IAutoPilotService private const int MaxReleaseNotesChars = 6000; private const string ReleaseNotesTruncatedMarker = "[Release notes truncated]"; - // Low temperature and a repeat penalty keep small models close to the source notes. - // The prod host runs Ollama on a CPU without AVX2 (~1-2 tokens/s), so the context and - // output budget stay small enough for a draft to finish well inside the HTTP timeout. + // Low temperature keeps small models close to the notes; small budgets suit a slow CPU-only host private static readonly OllamaGenerationOptions BlogGenerationOptions = new( Temperature: 0.2, TopP: 0.9, @@ -43,7 +41,6 @@ public AutoPilotService( [DisableConcurrentExecution(timeoutInSeconds: 600)] public async Task RunAutoPilotJobAsync() { - // Check if AutoPilot is enabled in DB settings var enabledSetting = (await _unitOfWork.SystemSettings.FindAsync(s => s.Key == "AutoPilot:Enabled")).FirstOrDefault(); if (enabledSetting != null && enabledSetting.Value.Equals("false", StringComparison.OrdinalIgnoreCase)) { @@ -51,7 +48,6 @@ public async Task RunAutoPilotJobAsync() return; } - // Read LLM model from DB, fall back to config/default var modelSetting = (await _unitOfWork.SystemSettings.FindAsync(s => s.Key == "AutoPilot:LlmModel")).FirstOrDefault(); var llmModel = modelSetting?.Value ?? _defaultLlmModel; @@ -91,7 +87,6 @@ public async Task RunAutoPilotJobAsync() if (!AutoPilotQualityGate.Passes(cleaned, targetPackage.PackageId, out var reason)) { // Returning instead of throwing: a Hangfire retry would only replay the same bad model output - // The model's own headings are logged so a rejection can be diagnosed without the full output var modelHeadings = llmResponse.Split('\n') .Select(l => l.Trim()) .Where(l => l.StartsWith('#') || l.StartsWith("**", StringComparison.Ordinal)) @@ -106,7 +101,7 @@ public async Task RunAutoPilotJobAsync() var post = BuildDraftPost(cleaned, targetPackage, latestReleaseNote, version, llmModel); await _unitOfWork.BlogPosts.AddAsync(post); - await _unitOfWork.SaveChangesAsync(); // Save blog to get the ID + await _unitOfWork.SaveChangesAsync(); _logger.LogInformation("AutoPilot: Created draft blog post for {PackageId}: {Slug}", targetPackage.PackageId, post.Slug); } @@ -119,11 +114,9 @@ public async Task RunAutoPilotJobAsync() private async Task<(CachedPackage? Package, CachedReleaseNote? ReleaseNote)> SelectInterestingPackageAsync() { - // Fetch 50 most recently searched / cached packages (include Registry for Value) var recentPackages = await _unitOfWork.CachedPackages.GetTopDescendingWithIncludeAsync( p => p.CachedAt, p => p.Registry, 50); - // Avoid duplicates: skip packages whose current version already has a blog post var allBlogs = await _unitOfWork.ToListAsync(_unitOfWork.BlogPosts.Query() .Select(b => new { b.PackageId, b.Version, b.CachedReleaseNoteId })); var bloggedVersionKeys = allBlogs @@ -135,20 +128,17 @@ public async Task RunAutoPilotJobAsync() .Select(b => b.CachedReleaseNoteId!.Value) .ToHashSet(); - // Find the first candidate that has meaningful release notes foreach (var package in recentPackages) { var releaseNote = await GetLatestReleaseNote(package.Id); if (releaseNote != null && bloggedReleaseNoteIds.Contains(releaseNote.Id)) { - // We've already written a blog post specifically for this release note continue; } string versionToCheck = releaseNote?.Version ?? package.LatestVersion ?? "Unknown"; - // Legacy / fallback checks by version string if (bloggedVersionKeys.Contains($"{package.PackageId}::{versionToCheck}") || bloggedVersionKeys.Contains($"{package.PackageId}::{package.LatestVersion}")) { @@ -227,9 +217,8 @@ private string GenerateSlug(string phrase) { string str = Regex.Replace(phrase.ToLowerInvariant(), @"[^a-z0-9\s-]", ""); str = Regex.Replace(str, @"[\s-]+", " ").Trim(); - str = Regex.Replace(str, @"\s", "-"); // hyphens + str = Regex.Replace(str, @"\s", "-"); - // add random hash to prevent conflicts str = $"{str}-{Guid.NewGuid().ToString().Substring(0, 5)}"; return str; } diff --git a/src/backend/PkgDiff.Application/Services/DependencyAuditService.cs b/src/backend/PkgDiff.Application/Services/DependencyAuditService.cs index 8a8f837..0ab35f3 100644 --- a/src/backend/PkgDiff.Application/Services/DependencyAuditService.cs +++ b/src/backend/PkgDiff.Application/Services/DependencyAuditService.cs @@ -10,9 +10,7 @@ namespace PkgDiff.Application.Services; -// A dependency audit: the visitor's packages are stored, a background job looks each one up in its -// registry (does it exist, what is its latest version), asks OSV about all of them in one batch and -// stores the report. Nothing is fetched while the visitor waits; the page polls the report. +// Audits run in a background job and are stored; nothing is fetched while the visitor waits public class DependencyAuditService : IDependencyAuditService { private static readonly TimeSpan Lifetime = TimeSpan.FromDays(30); @@ -207,7 +205,7 @@ private async Task LookUpAsync(IRegistryClient client, AuditDependenc } else { - // The registry's own spelling of the same version (NuGet writes 1.0 as 1.0.0) + // The registry's own spelling of the same version item.Version = PackageVersion.TryParse(client.VersionScheme, dependency.Version, out var parsed) ? versions.FirstOrDefault(v => v.Equals(parsed))?.Value ?? dependency.Version : dependency.Version; @@ -228,8 +226,7 @@ private async Task LookUpAsync(IRegistryClient client, AuditDependenc return item; } - // The lowest version that fixes every advisory: for each advisory its first fixed version above - // the checked one, and the highest of those + // Lowest version fixing every advisory: the highest first-fixed version above the checked one private static (string? Safe, bool HasUnfixed) SafeVersionOf(VersionScheme scheme, string current, List advisories) { if (!PackageVersion.TryParse(scheme, current, out var checkedVersion)) diff --git a/src/backend/PkgDiff.Application/Services/ExportService.cs b/src/backend/PkgDiff.Application/Services/ExportService.cs index 81f9f9c..99180dd 100644 --- a/src/backend/PkgDiff.Application/Services/ExportService.cs +++ b/src/backend/PkgDiff.Application/Services/ExportService.cs @@ -13,7 +13,6 @@ public class ExportService : IExportService // QuestPDF settings are process-wide: set them once, not on every request that resolves this scoped service static ExportService() { - // Configure QuestPDF license (Community license for open source projects) QuestPDF.Settings.License = LicenseType.Community; // Release notes carry arbitrary Unicode; a glyph missing from the bundled font must not fail the export @@ -42,7 +41,6 @@ public Task ExportToPdfAsync(PackageMetadata metadata) { column.Spacing(5); - // Package Info column.Item().Text($"Latest Version: {metadata.LatestVersion}").FontSize(12); if (!string.IsNullOrEmpty(metadata.Description)) { @@ -57,7 +55,6 @@ public Task ExportToPdfAsync(PackageMetadata metadata) ComposePackageSecurity(column, metadata); - // Versions foreach (var version in metadata.Versions) { column.Item().PaddingTop(15).Column(versionColumn => @@ -148,13 +145,11 @@ public Task ExportDiffToPdfAsync(AnalysisResult diff) { column.Spacing(5); - // Diff Info column.Item().Text($"From: {diff.FromVersion} → To: {diff.ToVersion}").FontSize(12); column.Item().Text($"Total Versions: {diff.TotalVersions}"); column.Item().PaddingTop(10).LineHorizontal(1).LineColor(Colors.Grey.Lighten2); - // Summary column.Item().PaddingTop(10).Text("Summary").SemiBold().FontSize(14); column.Item().PaddingTop(5).Table(table => { @@ -192,7 +187,6 @@ static IContainer CellStyle(IContainer container) ComposeUpgradeSecurity(column, diff); - // Versions column.Item().PaddingTop(10).Text("Changes by Version").SemiBold().FontSize(14); foreach (var version in diff.Versions) diff --git a/src/backend/PkgDiff.Application/Services/LiveUpdateService.cs b/src/backend/PkgDiff.Application/Services/LiveUpdateService.cs index afd26b5..ffac963 100644 --- a/src/backend/PkgDiff.Application/Services/LiveUpdateService.cs +++ b/src/backend/PkgDiff.Application/Services/LiveUpdateService.cs @@ -36,7 +36,7 @@ public async Task> GetLiveUpdatesAsync(int page = 1, page = Math.Max(page, 1); pageSize = Math.Clamp(pageSize, 1, MaxPageSize); - // A package cached before LatestVersionDate existed is ordered by its cache time until a refresh fills the date in + // Packages without a LatestVersionDate are ordered by cache time until a refresh fills it in var (items, total) = await _unitOfWork.CachedPackages.GetPageDescendingAsync( p => p.LatestVersion != "", p => p.LatestVersionDate ?? p.CachedAt, diff --git a/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs b/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs index addcad7..a8454d1 100644 --- a/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs +++ b/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs @@ -13,7 +13,6 @@ namespace PkgDiff.Application.Services; public class PackageAnalysisService : IPackageAnalysisService { - // A version whose notes were not found is looked up again after this long, not on every scrape of its package private static readonly TimeSpan EmptyNotesRetryAfter = TimeSpan.FromDays(1); private const int MaxScrapedVersions = 300; @@ -81,7 +80,6 @@ public async Task AnalyzePackageAsync(AnalyzeRequest request, st await SendProgress(connectionId, $"Initializing analysis for {request.PackageId}..."); await ValidatePackageExistsAsync(request, registry); - // No target version means "the latest one": the background refresh asks this way var latestRequested = string.IsNullOrEmpty(request.TargetVersion); await AutoSelectVersionsIfNeeded(request, registry); var (fromVersion, toVersion) = ParseVersions(request, registry); @@ -101,7 +99,7 @@ public async Task AnalyzePackageAsync(AnalyzeRequest request, st var summary = _analyzerService.GenerateSummary(versionInfoList); await SaveSearchHistoryAsync(request, versionInfoList, startTime, ipAddress); - // A version a visitor picked must not become news; it may only supply the date of the version already stored + // A visitor-picked version must not become news; it may only date the stored latest var isStoredLatest = registry.TryParseVersion(cachedPackage.LatestVersion, out var storedLatest) && storedLatest.Equals(toVersion); if (latestRequested || isStoredLatest) { @@ -142,8 +140,7 @@ await _workTracker.RunCoalescedAsync(WorkKeys.Scrape(registry, packageId), async string? githubRepoUrl = cachedPackage.GitHubRepositoryUrl; var registryClient = _registryFactory.GetClient(registry); - // The newest versions only: a package with thousands of them would hold a worker and the GitHub - // budget for hours. Older versions are still fetched when someone compares a range that has them. + // Newest versions only, so a package with thousands of them cannot hold a worker for hours var versions = (JsonSerializer.Deserialize>(cachedPackage.AllVersionsJson) ?? []) .Select(v => (Text: v, Parsed: registryClient.TryParseVersion(v, out var parsed) ? parsed : null)) .OrderByDescending(v => v.Parsed is not null) @@ -174,8 +171,7 @@ await _progressReporter.BroadcastBackgroundProgressAsync(packageId, } catch (GitHubRateLimitedException ex) { - // The remaining versions are not without notes: the scrape goes on once the limit resets, - // skipping the versions already done + // The scrape resumes once the limit resets, skipping the versions already done var delay = ex.ResetAt - DateTime.UtcNow + RateLimitResetMargin; if (delay < RateLimitResetMargin) delay = RateLimitResetMargin; @@ -217,7 +213,6 @@ public async Task> GetCachedPackagesAsync() return await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query() .Where(p => p.LatestVersion != "") .OrderByDescending(p => p.CachedAt) - // Anonymous and served whole: bounded, newest first .Take(MaxListedPackages) .Select(p => new PackageMetadata { @@ -252,7 +247,6 @@ public async Task> SearchSuggestionsAsync(string registr var registryId = await _cacheService.ResolveRegistryIdAsync(registry); var term = query.Trim().ToLowerInvariant(); - // Shortest names first: the closest match to what was typed return await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query() .Where(p => p.RegistryId == registryId && p.PackageId.ToLower().Contains(term)) .OrderBy(p => p.PackageId.Length) @@ -268,8 +262,6 @@ public async Task> SearchSuggestionsAsync(string registr })); } - // ==================== PRIVATE HELPER METHODS ==================== - private async Task SendProgress(string? connectionId, string message) { await _progressReporter.SendProgressAsync(connectionId, message); @@ -320,7 +312,6 @@ private async Task> ProcessVersionsAsync( var versionsToProcess = versionsList.Take(maxInitialVersions).ToList(); - // The cached notes of the whole range in one query, instead of one query per version var versionStrings = versionsToProcess.Select(v => v.ToString()).ToList(); var cachedNotes = (await _unitOfWork.CachedReleaseNotes .FindAsync(rn => rn.CachedPackageId == cachedPackage.Id && versionStrings.Contains(rn.Version))) @@ -376,7 +367,6 @@ private async Task> ProcessVersionsAsync( var versionInfo = CreateVersionInfo(version, releaseNotes, releaseNotesSource, publishedDate, cachedNote, isFromCache); versionInfoList.Add(versionInfo); - // Early stopping logic if (versionInfo.IsEmpty) { consecutiveEmpty++; @@ -423,8 +413,7 @@ private async Task ScrapeVersionReleaseNotesAsync( { var cachedNote = await _cacheService.GetCachedReleaseNoteAsync(cachedPackage.Id, versionString); - // Skip if already has actual release notes and not expired, or was looked up in vain only recently. - // A row seeded with the version list has no source yet: it was never looked up, however new it is. + // A row seeded from the version list has no source yet: it was never looked up var now = DateTime.UtcNow; if (cachedNote != null && cachedNote.ExpiresAt > now && (!cachedNote.IsEmpty @@ -436,7 +425,6 @@ private async Task ScrapeVersionReleaseNotesAsync( var releaseNotes = StoredMarkdown.Normalize(result.ReleaseNotes); PkgDiffTelemetry.RecordScrape(activity, registry.Id, result.Source, !string.IsNullOrWhiteSpace(releaseNotes)); - // Update githubRepoUrl if newly discovered if (result.DiscoveredGitHubUrl != null) { cachedPackage.GitHubRepositoryUrl = result.DiscoveredGitHubUrl; @@ -478,7 +466,6 @@ private async Task ScrapeVersionReleaseNotesAsync( await _cacheService.SaveReleaseNoteAsync(cachedNote); - // Extract GitHub @mentions and track contributors if (result.Source == "GitHub" && !string.IsNullOrWhiteSpace(releaseNotes)) { try @@ -543,8 +530,6 @@ private async Task SaveSearchHistoryAsync( DateTime startTime, string ipAddress) { - // Do not log internal background jobs (e.g. Hangfire scheduled tasks) into search history - // to prevent inflating metrics and obscuring real user data. if (ipAddress == "System") { return; @@ -581,7 +566,7 @@ private async Task SaveFailedScrapePlaceholderAsync(Domain.Entities.CachedPackag PublishedDate = null, IsEmpty = true, CachedAt = DateTime.UtcNow, - ExpiresAt = DateTime.UtcNow.AddDays(7) // Cache failure for 7 days + ExpiresAt = DateTime.UtcNow.AddDays(7) }; await _cacheService.SaveReleaseNoteAsync(cachedNote); } diff --git a/src/backend/PkgDiff.Application/Services/PackageCacheService.cs b/src/backend/PkgDiff.Application/Services/PackageCacheService.cs index a605a55..dd3a98e 100644 --- a/src/backend/PkgDiff.Application/Services/PackageCacheService.cs +++ b/src/backend/PkgDiff.Application/Services/PackageCacheService.cs @@ -88,7 +88,7 @@ public async Task GetOrCreateCachedPackageAsync( GitHubRepositoryUrl = pkgMeta?.GitHubRepositoryUrl, AllVersionsJson = JsonSerializer.Serialize(versionStrings), CachedAt = DateTime.UtcNow, - ExpiresAt = DateTime.UtcNow.AddDays(14) // Extended from 7 to 14 days + ExpiresAt = DateTime.UtcNow.AddDays(14) }; await _unitOfWork.CachedPackages.AddAsync(cachedPackage); @@ -266,19 +266,17 @@ private async Task SaveFreshMetadataToCache( cached.GitHubRepositoryUrl = metadata.GitHubRepositoryUrl; cached.AllVersionsJson = JsonSerializer.Serialize(metadata.Versions.Select(v => v.Version)); cached.CachedAt = DateTime.UtcNow; - cached.ExpiresAt = DateTime.UtcNow.AddDays(14); // Extended from 7 to 14 days + cached.ExpiresAt = DateTime.UtcNow.AddDays(14); await _unitOfWork.CachedPackages.UpdateAsync(cached); } await _unitOfWork.SaveChangesAsync(); - // What the registry itself calls latest: stored with its publish date, and pushed to connected visitors when it is new var latestVersion = metadata.LatestVersion ?? string.Empty; var latestPublished = metadata.Versions.FirstOrDefault(v => v.Version == latestVersion)?.PublishedDate; await _liveUpdates.RecordLatestVersionAsync(cached, registry, latestVersion, latestPublished, authoritative: true); - // Seed CachedReleaseNotes with version metadata. A concurrent request may seed some of the same - // versions first; then the rows that are still missing are added in a second pass. + // A concurrent request may seed some versions first; the missing rows are added in a second pass for (var attempt = 0; ; attempt++) { var existingNotes = (await _unitOfWork.CachedReleaseNotes @@ -299,7 +297,7 @@ await _unitOfWork.CachedReleaseNotes.AddAsync(new CachedReleaseNote PublishedDate = v.PublishedDate, IsEmpty = true, CachedAt = DateTime.UtcNow, - ExpiresAt = DateTime.UtcNow.AddDays(90) // Extended from 30 to 90 days - release notes rarely change + ExpiresAt = DateTime.UtcNow.AddDays(90) }); } diff --git a/src/backend/PkgDiff.Application/Services/PackageTrustService.cs b/src/backend/PkgDiff.Application/Services/PackageTrustService.cs index 48d17b1..f873c2d 100644 --- a/src/backend/PkgDiff.Application/Services/PackageTrustService.cs +++ b/src/backend/PkgDiff.Application/Services/PackageTrustService.cs @@ -9,21 +9,17 @@ namespace PkgDiff.Application.Services; -// Security data is never fetched while a visitor waits: a request reads what is stored and, when -// that is missing or old, queues a background refresh. The refresh asks the providers, stores the -// result and tells the open pages over SignalR. +// Security data is never fetched inline: requests read stored data and queue a background refresh public class PackageTrustService : IPackageTrustService { private const int DefaultCacheHours = 24; private const int MinCacheHours = 1; private const int MaxCacheHours = 168; - // OSV answers up to 1000 queries per batch; a package with more versions is checked for its newest 1000 + // OSV takes at most 1000 queries per batch; larger packages are checked for the newest 1000 private const int MaxVersionsChecked = 1000; private const int ExpiredPerRun = 100; private const int MaxErrorLength = 1000; - // One refresh per package is queued at most this often, however many visitors open it private static readonly TimeSpan RequeueAfter = TimeSpan.FromMinutes(10); - // A failed refresh is tried again after this long instead of on the next visit private static readonly TimeSpan RetryAfterFailure = TimeSpan.FromHours(1); private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web); diff --git a/src/backend/PkgDiff.Application/Services/RoleAssignmentRules.cs b/src/backend/PkgDiff.Application/Services/RoleAssignmentRules.cs index a6c9156..e361e82 100644 --- a/src/backend/PkgDiff.Application/Services/RoleAssignmentRules.cs +++ b/src/backend/PkgDiff.Application/Services/RoleAssignmentRules.cs @@ -31,8 +31,7 @@ public static (bool Allowed, int StatusCode, string? Error) Evaluate( return (true, 200, null); } - // Permission policies pass on the claim alone, so a non-Admin must never obtain (through a role they assign, - // or an account they take over) a permission they do not already hold + // Policies check the claim alone, so a non-Admin must never obtain a permission they lack public static (bool Allowed, int StatusCode, string? Error) EvaluatePermissionSubset( IEnumerable permissions, bool callerIsAdmin, diff --git a/src/backend/PkgDiff.Application/Services/VersionFilterService.cs b/src/backend/PkgDiff.Application/Services/VersionFilterService.cs index c6af243..b3810d4 100644 --- a/src/backend/PkgDiff.Application/Services/VersionFilterService.cs +++ b/src/backend/PkgDiff.Application/Services/VersionFilterService.cs @@ -21,14 +21,12 @@ public VersionFilterService(IUnitOfWork unitOfWork, IMemoryCache cache, ILogger< public async Task> GetActiveExcludePatternsAsync() { - // Try to get from cache first if (_cache.TryGetValue(CacheKey, out IEnumerable? cachedPatterns) && cachedPatterns != null) { _logger.LogDebug("[VersionFilter] Using cached patterns: {Patterns}", string.Join(", ", cachedPatterns)); return cachedPatterns; } - // Fetch from database var patterns = await _unitOfWork.VersionExcludePatterns.GetAllAsync(); var activePatterns = patterns .Where(p => p.IsActive) @@ -38,7 +36,6 @@ public async Task> GetActiveExcludePatternsAsync() _logger.LogDebug("[VersionFilter] Loaded {Count} active patterns from DB: {Patterns}", activePatterns.Count, string.Join(", ", activePatterns)); - // Cache the result _cache.Set(CacheKey, activePatterns, CacheExpiration); return activePatterns; @@ -49,7 +46,6 @@ public bool ShouldExcludeVersion(string version, IEnumerable patterns) if (string.IsNullOrWhiteSpace(version)) return false; - // Check if version contains any of the exclude patterns (case-insensitive) var shouldExclude = patterns.Any(pattern => version.Contains(pattern, StringComparison.OrdinalIgnoreCase)); @@ -63,10 +59,8 @@ public bool ShouldExcludeVersion(string version, IEnumerable patterns) public async Task RefreshPatternsAsync() { - // Remove from cache to force refresh on next request _cache.Remove(CacheKey); - // Pre-load the cache await GetActiveExcludePatternsAsync(); } } diff --git a/src/backend/PkgDiff.Application/Services/WorkCancellation.cs b/src/backend/PkgDiff.Application/Services/WorkCancellation.cs index ddad4b7..628c67d 100644 --- a/src/backend/PkgDiff.Application/Services/WorkCancellation.cs +++ b/src/backend/PkgDiff.Application/Services/WorkCancellation.cs @@ -1,9 +1,7 @@ namespace PkgDiff.Application.Services; -// The cancellation token of the work that is running on this async flow. An endpoint that starts -// long work sets it from the request; the outbound HTTP handler, the repository reads and the rate -// limiter honour it, so the work stops without every method in between taking a token parameter. -// Outside such a scope (background jobs, short requests) the token is None. +// Ambient cancellation token of the long work a request started; None outside such a scope. +// Outbound HTTP, repository reads and the rate limiter honour it without a token parameter. public static class WorkCancellation { private static readonly AsyncLocal Current = new(); diff --git a/src/backend/PkgDiff.Application/Versioning/PackageVersion.cs b/src/backend/PkgDiff.Application/Versioning/PackageVersion.cs index 21f93ca..01c1ebd 100644 --- a/src/backend/PkgDiff.Application/Versioning/PackageVersion.cs +++ b/src/backend/PkgDiff.Application/Versioning/PackageVersion.cs @@ -3,7 +3,6 @@ namespace PkgDiff.Application.Versioning; -// A version of one package, parsed and ordered by the scheme of its registry public sealed class PackageVersion : IComparable, IEquatable { // Either a Pep440Version or a NuGetVersion; versions of one package always share the kind diff --git a/src/backend/PkgDiff.Application/Versioning/VersionScheme.cs b/src/backend/PkgDiff.Application/Versioning/VersionScheme.cs index e0ef4f8..5c85ec2 100644 --- a/src/backend/PkgDiff.Application/Versioning/VersionScheme.cs +++ b/src/backend/PkgDiff.Application/Versioning/VersionScheme.cs @@ -1,6 +1,5 @@ namespace PkgDiff.Application.Versioning; -// The rules a registry's versions are parsed and ordered by public enum VersionScheme { // NuGet's own rules; a version is spelled the way NuGet normalizes it diff --git a/src/backend/PkgDiff.Domain/Common/DuplicateKeyException.cs b/src/backend/PkgDiff.Domain/Common/DuplicateKeyException.cs index 49b3f58..4bfe94b 100644 --- a/src/backend/PkgDiff.Domain/Common/DuplicateKeyException.cs +++ b/src/backend/PkgDiff.Domain/Common/DuplicateKeyException.cs @@ -1,6 +1,5 @@ namespace PkgDiff.Domain.Common; -// A save was refused because a row with the same unique key already exists public sealed class DuplicateKeyException : Exception { public DuplicateKeyException(Exception? innerException = null) diff --git a/src/backend/PkgDiff.Domain/Entities/BlogPost.cs b/src/backend/PkgDiff.Domain/Entities/BlogPost.cs index 36d2538..76c7090 100644 --- a/src/backend/PkgDiff.Domain/Entities/BlogPost.cs +++ b/src/backend/PkgDiff.Domain/Entities/BlogPost.cs @@ -15,7 +15,6 @@ public class BlogPost : BaseEntity public DateTime? PublishedAt { get; set; } public string LlmModelUsed { get; set; } = string.Empty; - // Optional relation back to the release note this blog was generated from public int? CachedReleaseNoteId { get; set; } public CachedReleaseNote? CachedReleaseNote { get; set; } } diff --git a/src/backend/PkgDiff.Domain/Entities/CachedPackage.cs b/src/backend/PkgDiff.Domain/Entities/CachedPackage.cs index a654511..ed17468 100644 --- a/src/backend/PkgDiff.Domain/Entities/CachedPackage.cs +++ b/src/backend/PkgDiff.Domain/Entities/CachedPackage.cs @@ -8,22 +8,20 @@ public class CachedPackage : BaseEntity public int RegistryId { get; set; } public string LatestVersion { get; set; } = string.Empty; public string? PreviousLatestVersion { get; set; } - public DateTime? LatestVersionDate { get; set; } // Publish date of LatestVersion, or when it was first seen if the registry gave none + public DateTime? LatestVersionDate { get; set; } // Falls back to first-seen time when the registry gives none public string? Description { get; set; } public string? Authors { get; set; } public string? IconUrl { get; set; } public string? ProjectUrl { get; set; } - public string AllVersionsJson { get; set; } = "[]"; // JSON serialized list + public string AllVersionsJson { get; set; } = "[]"; public string? GitHubRepositoryUrl { get; set; } - public string? ReleaseNotesSource { get; set; } // "NuGet" or "GitHub" or "PubDev" + public string? ReleaseNotesSource { get; set; } public DateTime CachedAt { get; set; } public DateTime ExpiresAt { get; set; } - // When a visitor last opened or compared the package (at most hourly); null until then. Only packages - // looked at recently are refreshed in the background. + // Updated at most hourly; only recently viewed packages are refreshed in the background public DateTime? LastViewedAt { get; set; } - // Navigation properties public Registry Registry { get; set; } = null!; public ICollection ReleaseNotes { get; set; } = new List(); } diff --git a/src/backend/PkgDiff.Domain/Entities/CachedReleaseNote.cs b/src/backend/PkgDiff.Domain/Entities/CachedReleaseNote.cs index 881f2e8..c2f5fbb 100644 --- a/src/backend/PkgDiff.Domain/Entities/CachedReleaseNote.cs +++ b/src/backend/PkgDiff.Domain/Entities/CachedReleaseNote.cs @@ -7,13 +7,12 @@ public class CachedReleaseNote : BaseEntity public int CachedPackageId { get; set; } public string Version { get; set; } = string.Empty; public string ReleaseNotes { get; set; } = string.Empty; - public string? ParsedNotesJson { get; set; } // JSON array of {category, text} - public string? ReleaseNotesSource { get; set; } // "NuGet" or "GitHub" + public string? ParsedNotesJson { get; set; } + public string? ReleaseNotesSource { get; set; } public DateTime? PublishedDate { get; set; } public bool IsEmpty { get; set; } public DateTime CachedAt { get; set; } public DateTime ExpiresAt { get; set; } - // Navigation property public CachedPackage CachedPackage { get; set; } = null!; } diff --git a/src/backend/PkgDiff.Domain/Entities/DependencyAudit.cs b/src/backend/PkgDiff.Domain/Entities/DependencyAudit.cs index d8e44bd..17b2635 100644 --- a/src/backend/PkgDiff.Domain/Entities/DependencyAudit.cs +++ b/src/backend/PkgDiff.Domain/Entities/DependencyAudit.cs @@ -2,9 +2,7 @@ namespace PkgDiff.Domain.Entities; -// One dependency audit: the packages a visitor pasted and, once the background job is done, the -// report. Found by PublicId (a random GUID) so a report link cannot be guessed. The pasted file -// itself is never stored, only the package names and versions taken from it. +// Found by random PublicId so report links cannot be guessed; the pasted file is never stored public class DependencyAudit : BaseEntity { public Guid PublicId { get; set; } diff --git a/src/backend/PkgDiff.Domain/Entities/PackageSecurityInfo.cs b/src/backend/PkgDiff.Domain/Entities/PackageSecurityInfo.cs index 9444a5a..7382d20 100644 --- a/src/backend/PkgDiff.Domain/Entities/PackageSecurityInfo.cs +++ b/src/backend/PkgDiff.Domain/Entities/PackageSecurityInfo.cs @@ -2,12 +2,11 @@ namespace PkgDiff.Domain.Entities; -// Security data of one package as the background job last collected it. The data itself is one -// JSON document (the PackageTrust DTO): it is always read and written whole. +// The PackageTrust DTO as one JSON document, always read and written whole public class PackageSecurityInfo : BaseEntity { public string Registry { get; set; } = string.Empty; - // Lower-case, so "Newtonsoft.Json" and "newtonsoft.json" share one row + // Lower-cased so ids differing only in case share one row public string PackageKey { get; set; } = string.Empty; // The spelling the package was first asked for with, which is how its cache row is found public string PackageId { get; set; } = string.Empty; diff --git a/src/backend/PkgDiff.Domain/Entities/SocialMediaPost.cs b/src/backend/PkgDiff.Domain/Entities/SocialMediaPost.cs index 5dd1b2d..ff8cc54 100644 --- a/src/backend/PkgDiff.Domain/Entities/SocialMediaPost.cs +++ b/src/backend/PkgDiff.Domain/Entities/SocialMediaPost.cs @@ -24,7 +24,6 @@ public class SocialMediaPost : BaseEntity public string? ErrorMessage { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; - // Optional reference to a blog post if it was auto-generated public int? BlogPostId { get; set; } public BlogPost? BlogPost { get; set; } } diff --git a/src/backend/PkgDiff.Domain/Entities/User.cs b/src/backend/PkgDiff.Domain/Entities/User.cs index 0a6d540..0e7adad 100644 --- a/src/backend/PkgDiff.Domain/Entities/User.cs +++ b/src/backend/PkgDiff.Domain/Entities/User.cs @@ -4,7 +4,6 @@ namespace PkgDiff.Domain.Entities; public class ApplicationUser : IdentityUser { - // Add custom properties if needed public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime? LastLoginAt { get; set; } } diff --git a/src/backend/PkgDiff.Domain/Interfaces/IUnitOfWork.cs b/src/backend/PkgDiff.Domain/Interfaces/IUnitOfWork.cs index c7caee3..475b591 100644 --- a/src/backend/PkgDiff.Domain/Interfaces/IUnitOfWork.cs +++ b/src/backend/PkgDiff.Domain/Interfaces/IUnitOfWork.cs @@ -21,7 +21,6 @@ public interface IUnitOfWork : IDisposable Task SaveChangesAsync(); - // Run a read-only query composed on IRepository.Query() Task> ToListAsync(IQueryable query); Task CountAsync(IQueryable query); Task BeginTransactionAsync(); diff --git a/src/backend/PkgDiff.Domain/Interfaces/IVersionFilterService.cs b/src/backend/PkgDiff.Domain/Interfaces/IVersionFilterService.cs index eac1893..7460ce6 100644 --- a/src/backend/PkgDiff.Domain/Interfaces/IVersionFilterService.cs +++ b/src/backend/PkgDiff.Domain/Interfaces/IVersionFilterService.cs @@ -2,18 +2,9 @@ namespace PkgDiff.Domain.Interfaces; public interface IVersionFilterService { - /// - /// Gets all active exclude patterns from database (cached). - /// Task> GetActiveExcludePatternsAsync(); - /// - /// Checks if a version should be excluded based on patterns. - /// bool ShouldExcludeVersion(string version, IEnumerable patterns); - /// - /// Refreshes the cached patterns from database. - /// Task RefreshPatternsAsync(); } diff --git a/src/backend/PkgDiff.Infrastructure/Data/ApplicationDbContext.cs b/src/backend/PkgDiff.Infrastructure/Data/ApplicationDbContext.cs index 942b19b..f46fcab 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/ApplicationDbContext.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/ApplicationDbContext.cs @@ -35,9 +35,8 @@ protected override void ConfigureConventions(ModelConfigurationBuilder configura protected override void OnModelCreating(ModelBuilder modelBuilder) { - base.OnModelCreating(modelBuilder); // CRITICAL: Call base for Identity tables + base.OnModelCreating(modelBuilder); - // Apply all configurations from assembly modelBuilder.ApplyConfigurationsFromAssembly(typeof(ApplicationDbContext).Assembly); } } diff --git a/src/backend/PkgDiff.Infrastructure/Data/Configurations/CachedPackageConfiguration.cs b/src/backend/PkgDiff.Infrastructure/Data/Configurations/CachedPackageConfiguration.cs index 7a235be..8d2c48f 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/Configurations/CachedPackageConfiguration.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/Configurations/CachedPackageConfiguration.cs @@ -44,7 +44,6 @@ public void Configure(EntityTypeBuilder builder) .IsRequired() .HasColumnType("text"); - // Unique index on PackageId and RegistryId builder.HasIndex(x => new { x.PackageId, x.RegistryId }) .IsUnique(); @@ -54,7 +53,6 @@ public void Configure(EntityTypeBuilder builder) // Index for the Live Updates feed order builder.HasIndex(x => x.LatestVersionDate); - // Navigation builder.HasMany(x => x.ReleaseNotes) .WithOne(x => x.CachedPackage) .HasForeignKey(x => x.CachedPackageId) diff --git a/src/backend/PkgDiff.Infrastructure/Data/Configurations/SearchHistoryConfiguration.cs b/src/backend/PkgDiff.Infrastructure/Data/Configurations/SearchHistoryConfiguration.cs index baf2ddf..5659206 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/Configurations/SearchHistoryConfiguration.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/Configurations/SearchHistoryConfiguration.cs @@ -25,7 +25,6 @@ public void Configure(EntityTypeBuilder builder) builder.Property(x => x.IpAddress) .HasMaxLength(45); // IPv6 max length - // Indexes for common queries builder.HasIndex(x => x.PackageId); builder.HasIndex(x => x.SearchedAt); } diff --git a/src/backend/PkgDiff.Infrastructure/Data/Configurations/SystemSettingsConfiguration.cs b/src/backend/PkgDiff.Infrastructure/Data/Configurations/SystemSettingsConfiguration.cs index cdd7083..6335315 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/Configurations/SystemSettingsConfiguration.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/Configurations/SystemSettingsConfiguration.cs @@ -24,7 +24,6 @@ public void Configure(EntityTypeBuilder builder) builder.Property(x => x.DataType) .HasMaxLength(50); - // Unique index on Key builder.HasIndex(x => x.Key).IsUnique(); } } diff --git a/src/backend/PkgDiff.Infrastructure/Data/Configurations/VersionExcludePatternConfiguration.cs b/src/backend/PkgDiff.Infrastructure/Data/Configurations/VersionExcludePatternConfiguration.cs index dd53465..13ac08e 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/Configurations/VersionExcludePatternConfiguration.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/Configurations/VersionExcludePatternConfiguration.cs @@ -21,7 +21,6 @@ public void Configure(EntityTypeBuilder builder) .IsRequired() .HasDefaultValue(true); - // Seed data with common exclude patterns builder.HasData( new VersionExcludePattern { Id = 1, Pattern = "dev", Description = "Development builds (e.g., 1.0.0-dev-123)", IsActive = true }, new VersionExcludePattern { Id = 2, Pattern = "ci", Description = "Continuous Integration builds (e.g., 1.0.0-ci-456)", IsActive = true }, diff --git a/src/backend/PkgDiff.Infrastructure/Data/SystemSettingsMaintenance.cs b/src/backend/PkgDiff.Infrastructure/Data/SystemSettingsMaintenance.cs index f2f1063..e97ce88 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/SystemSettingsMaintenance.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/SystemSettingsMaintenance.cs @@ -48,8 +48,7 @@ public static int Apply(IEnumerable rows, ISecretProtector prote } } - // An already-prefixed value is never touched, whichever key produced it: a value that - // cannot be decrypted (rotated or lost key) must survive until an admin re-enters it. + // Prefixed values are never touched: an undecryptable one must survive until an admin re-enters it if (row.IsSecret && !string.IsNullOrEmpty(row.Value) && !protector.IsProtected(row.Value)) { row.Value = protector.Protect(row.Value); diff --git a/src/backend/PkgDiff.Infrastructure/Data/UtcDateTimeConverter.cs b/src/backend/PkgDiff.Infrastructure/Data/UtcDateTimeConverter.cs index 0171ba8..9b0bf8c 100644 --- a/src/backend/PkgDiff.Infrastructure/Data/UtcDateTimeConverter.cs +++ b/src/backend/PkgDiff.Infrastructure/Data/UtcDateTimeConverter.cs @@ -2,8 +2,7 @@ namespace PkgDiff.Infrastructure.Data; -// Every DateTime column holds UTC. A value without a kind is taken as UTC, which is what the columns -// have always held; a local one is converted. What is read comes back as UTC. +// Columns hold UTC: a value without a kind is taken as UTC, a local one converted. public class UtcDateTimeConverter : ValueConverter { public UtcDateTimeConverter() diff --git a/src/backend/PkgDiff.Infrastructure/Repositories/UnitOfWork.cs b/src/backend/PkgDiff.Infrastructure/Repositories/UnitOfWork.cs index b892ae3..da51eac 100644 --- a/src/backend/PkgDiff.Infrastructure/Repositories/UnitOfWork.cs +++ b/src/backend/PkgDiff.Infrastructure/Repositories/UnitOfWork.cs @@ -62,8 +62,7 @@ public async Task SaveChangesAsync() } catch (DbUpdateException ex) when (ex.InnerException is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }) { - // The whole batch was rolled back. The rows it tried to insert are forgotten, or the next - // save of this context would send them again and fail the same way. + // The batch was rolled back: forget its inserts, or the next save would resend them and fail foreach (var entry in _context.ChangeTracker.Entries().Where(e => e.State == EntityState.Added).ToList()) entry.State = EntityState.Detached; diff --git a/src/backend/PkgDiff.Infrastructure/Security/HtmlNestingGuard.cs b/src/backend/PkgDiff.Infrastructure/Security/HtmlNestingGuard.cs index 3a503f5..4dab9cd 100644 --- a/src/backend/PkgDiff.Infrastructure/Security/HtmlNestingGuard.cs +++ b/src/backend/PkgDiff.Infrastructure/Security/HtmlNestingGuard.cs @@ -1,10 +1,7 @@ namespace PkgDiff.Infrastructure.Security; -// AngleSharp builds and walks the DOM recursively: a page nested some thousands of elements deep -// overflows the stack while it is parsed or its text is read, and a stack overflow ends the whole -// process. Pages come from addresses package authors choose, so their depth is measured first, in -// one pass and without recursion. Elements whose end tag may be left out (p, li, td...) and void -// elements do not count, so ordinary sloppy HTML stays well below the limit. +// AngleSharp recurses over the DOM, so a deeply nested page overflows the stack and kills the process. +// Depth is measured first without recursion; optional-end-tag and void elements do not count. public static class HtmlNestingGuard { public const int MaxDepth = 512; diff --git a/src/backend/PkgDiff.Infrastructure/Security/SsrfGuard.cs b/src/backend/PkgDiff.Infrastructure/Security/SsrfGuard.cs index 7014ea0..edd9d03 100644 --- a/src/backend/PkgDiff.Infrastructure/Security/SsrfGuard.cs +++ b/src/backend/PkgDiff.Infrastructure/Security/SsrfGuard.cs @@ -31,8 +31,7 @@ public static SocketsHttpHandler CreateHandler( MaxAutomaticRedirections = MaxRedirects, PooledConnectionLifetime = TimeSpan.FromMinutes(2), AutomaticDecompression = DecompressionMethods.All, - // Every new host (including redirect targets) re-enters this callback, and the socket - // connects only to the addresses validated here, so DNS rebinding cannot swap the target. + // Every host, redirects included, re-enters here and connects only to validated addresses ConnectCallback = async (context, cancellationToken) => { var endPoint = context.DnsEndPoint; diff --git a/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs b/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs index 530099e..4792958 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs @@ -6,7 +6,7 @@ namespace PkgDiff.Infrastructure.Services; public class AnalyzerService : IAnalyzerService { - // Line-start prefix patterns → category mapping (most specific first) + // Ordered most specific first private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefixRules = [ (["breaking change", "breaking"], ChangeCategory.BreakingChange), @@ -18,7 +18,6 @@ private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefi (["change", "changed", "update", "updated", "improvement", "enhanced", "enhancement"], ChangeCategory.Other), ]; - // Fallback keyword matching (for release notes without structured tags) private static readonly (string[] Keywords, ChangeCategory Category)[] KeywordRules = [ (["breaking", "incompatible"], ChangeCategory.BreakingChange), @@ -27,11 +26,6 @@ private static readonly (string[] Keywords, ChangeCategory Category)[] KeywordRu (["performance", "optimize", "faster"], ChangeCategory.Performance), ]; - /// - /// Parses release notes into per-line entries with category detection. - /// Detects tags at line start (e.g., "Fix - ...", "New feature - ...") and strips them from text. - /// Tags mid-sentence are NOT removed. - /// public List ParseReleaseNotes(string releaseNotes) { if (string.IsNullOrWhiteSpace(releaseNotes)) @@ -46,7 +40,6 @@ public List ParseReleaseNotes(string releaseNotes) if (string.IsNullOrWhiteSpace(line)) continue; - // Strip markdown list markers (-, *, •) var cleaned = StripListMarker(line); var (category, text) = DetectAndStripPrefix(cleaned); @@ -64,10 +57,6 @@ public List ParseReleaseNotes(string releaseNotes) return entries; } - /// - /// Returns distinct categories found in release notes. - /// Uses line-start tag detection first, falls back to keyword matching. - /// public List CategorizeReleaseNotes(string releaseNotes) { if (string.IsNullOrWhiteSpace(releaseNotes)) @@ -79,7 +68,6 @@ public List CategorizeReleaseNotes(string releaseNotes) .Where(c => c != ChangeCategory.Other) .ToHashSet(); - // If no specific categories found from line prefixes, try keyword fallback if (categories.Count == 0) { var lower = releaseNotes.ToLowerInvariant(); @@ -133,10 +121,6 @@ public Summary GenerateSummary(List versions) return summary; } - /// - /// Detects a category tag at the start of a line and strips it. - /// Returns (category, cleanedText). If no tag found, returns (Other, originalText). - /// private static (ChangeCategory Category, string Text) DetectAndStripPrefix(string line) { var lower = line.ToLowerInvariant(); @@ -150,7 +134,6 @@ private static (ChangeCategory Category, string Text) DetectAndStripPrefix(strin var rest = line[prefix.Length..].TrimStart(); - // Must be followed by a separator (-, :, –, —) or be the entire line if (rest.Length == 0) return (category, string.Empty); @@ -160,14 +143,11 @@ private static (ChangeCategory Category, string Text) DetectAndStripPrefix(strin return (category, text); } - // Also handle "Fix #123" or "Fixed something" patterns - // (tag directly followed by description without separator) if (rest[0] is ' ' or '#') return (category, rest.TrimStart()); } } - // Check for markdown heading patterns: ### Bug Fixes, ## New Features if (lower.StartsWith('#')) { var heading = line.TrimStart('#').Trim(); @@ -176,25 +156,20 @@ private static (ChangeCategory Category, string Text) DetectAndStripPrefix(strin foreach (var (prefixes, category) in LinePrefixRules) { if (prefixes.Any(p => headingLower.StartsWith(p))) - return (category, string.Empty); // Headings are section titles, no text to keep + return (category, string.Empty); } } return (ChangeCategory.Other, line); } - /// - /// Strips markdown list markers (-, *, •, numbered) from the start of a line. - /// private static string StripListMarker(string line) { var trimmed = line.TrimStart(); - // "- text", "* text", "• text" if (trimmed.Length > 1 && trimmed[0] is '-' or '*' or '•' && trimmed[1] == ' ') return trimmed[2..].TrimStart(); - // "1. text", "2) text" var i = 0; while (i < trimmed.Length && char.IsDigit(trimmed[i])) i++; if (i > 0 && i < trimmed.Length && trimmed[i] is '.' or ')') @@ -207,10 +182,8 @@ private static string StripListMarker(string line) return trimmed; } - // Common annotations, framework keywords, and bot names that are NOT GitHub usernames private static readonly HashSet _mentionBlocklist = new(StringComparer.OrdinalIgnoreCase) { - // JSDoc / JavaDoc annotations "param", "returns", "return", "deprecated", "since", "see", "throws", "throw", "type", "typedef", "property", "prop", "readonly", "example", "override", "ignore", "event", "callback", "fires", "listens", "constructor", "extends", @@ -223,37 +196,26 @@ private static string StripListMarker(string line) "private", "protected", "public", "requires", "static", "summary", "this", "todo", "tutorial", "variation", "version", "yields", "yield", - // CSS / SCSS / PostCSS directives "media", "import", "keyframes", "include", "mixin", "extend", "use", "forward", "each", "for", "while", "if", "else", "function", "return", "charset", "namespace", "supports", "page", "font-face", "viewport", "layer", "apply", "tailwind", "screen", "responsive", - // Common GitHub bots / CI accounts (add more as needed) "dependabot", "renovate", "renovate-bot", "github-actions", "actions-user", "snyk-bot", "greenkeeper", "semantic-release-bot", "allcontributors", "renovateapp", "imgbot", "deepsource-autofix", - // npm scoped packages (e.g. @types/node, @babel/core) — org names not people "types", "babel", "angular", "storybook", "nestjs", "vitest", - // Email domains commonly appearing in @user@domain.com patterns "gmail", "outlook", "hotmail", "yahoo", "icloud", - // JavaScript / Node.js built-in symbols, methods, and reserved words "match", "iterator", "dispose", "species", "latest", "next", "current", "deprecated", "experimental", "stable", "beta", "alpha", "rc", - // Misc words often used in changelog prose "here", "everyone", "all", "team", "me", "you", "us", "channel", "mention", "thanks", "thank", }; - /// - /// Extracts GitHub @username mentions from release note text. - /// Valid GitHub usernames: alphanumeric and hyphens, 1-39 chars. - /// Filters out JSDoc/CSS annotations, known bots, and email-like patterns. - /// private const int MaxMentionsPerNote = 20; public IEnumerable ExtractMentions(string text) @@ -264,13 +226,13 @@ public IEnumerable ExtractMentions(string text) var matches = Regex.Matches(text, @"@([a-zA-Z0-9](?:[a-zA-Z0-9\-]{0,37}[a-zA-Z0-9])?)", RegexOptions.Compiled); return matches .Select(m => m.Groups[1].Value) - .Where(u => u.Length >= 3) // GH minimum meaningful length (filters '0','1','qq' etc) - .Where(u => !System.Text.RegularExpressions.Regex.IsMatch(u, @"^\d+$")) // skip pure numbers (e.g. version refs like @16, @18) - .Where(u => !u.Contains('.')) // skip email-like patterns - .Where(u => !_mentionBlocklist.Contains(u)) // skip annotations & bots - .Where(u => !u.StartsWith('-') && !u.EndsWith('-')) // invalid username chars + .Where(u => u.Length >= 3) + .Where(u => !System.Text.RegularExpressions.Regex.IsMatch(u, @"^\d+$")) + .Where(u => !u.Contains('.')) + .Where(u => !_mentionBlocklist.Contains(u)) + .Where(u => !u.StartsWith('-') && !u.EndsWith('-')) .Distinct(StringComparer.OrdinalIgnoreCase) - // Each mention costs a GitHub lookup and the text is written by the package author + // Each mention costs a GitHub lookup and the text is untrusted .Take(MaxMentionsPerNote); } } diff --git a/src/backend/PkgDiff.Infrastructure/Services/ContributorService.cs b/src/backend/PkgDiff.Infrastructure/Services/ContributorService.cs index 6765744..fb1c3b4 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ContributorService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ContributorService.cs @@ -45,7 +45,6 @@ public async Task> GetLeaderboardAsync(int limit = 100) .Take(pageSize) .ToListAsync(); - // Global rank = position in full ordered set var offset = (page - 1) * pageSize; var items = contributors .Select((c, i) => new ContributorDto( @@ -65,7 +64,6 @@ public async Task UpsertContributorMentionsAsync(IEnumerable usernames) foreach (var username in usernames) { - // Validate against GitHub API - skip if not a real user if (!await _validator.IsValidGitHubUserAsync(username)) continue; @@ -109,7 +107,6 @@ public async Task BackfillFromExistingReleaseNotesAsync() var mentions = _analyzerService.ExtractMentions(note.ReleaseNotes); foreach (var username in mentions) { - // Validate against GitHub API during backfill too if (!await _validator.IsValidGitHubUserAsync(username)) continue; diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubChangelogReader.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubChangelogReader.cs index 3dd207c..76ff4f4 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubChangelogReader.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubChangelogReader.cs @@ -7,8 +7,7 @@ namespace PkgDiff.Infrastructure.Services; -// Finds the notes of a version in the changelog file of a repository, for releases whose own notes -// are empty or only point to the changelog +// Reads a version's notes from the repo changelog when its release notes are empty or a link public partial class GitHubChangelogReader { private static readonly string[] CandidateFiles = @@ -56,15 +55,12 @@ public static bool IsRedirection(string notes) { if (string.IsNullOrWhiteSpace(notes)) return false; - // If notes are very long, they probably contain actual notes alongside a link if (notes.Length > 250) return false; var lower = notes.ToLowerInvariant(); - // Check if it contains changelog/history/changes reference if (lower.Contains("changelog") || lower.Contains("history") || lower.Contains("changes")) { - // Check for redirection indicators if (lower.Contains("see") || lower.Contains("refer") || lower.Contains("view") || @@ -79,8 +75,7 @@ public static bool IsRedirection(string notes) return false; } - // Where a changelog can be: the folder of the package in a monorepo (from the repository URL, - // e.g. .../tree/main/packages/core), the repository root, and the repository wiki + // Looks in the package's monorepo folder, then the repository root, then the wiki public async Task FetchAsync(string githubRepoUrl, string version, string? packageId = null) { var (owner, repo) = GitHubUrls.Parse(githubRepoUrl, _logger); @@ -104,25 +99,21 @@ public static bool IsRedirection(string notes) var knowledge = _cache.Changelog(repoKey); IReadOnlyList changelogFiles = CandidateFiles; - // Try API first to get exact list of existing changelog files await ResolveExistingFilesViaApiAsync(owner, repo, folder, repoKey, knowledge); var existingFiles = knowledge.ExistingFiles(); if (knowledge.IsFullyResolved && existingFiles != null) { - // If the cache says there are no changelog files, we can skip immediately! if (existingFiles.Count == 0) { _logger.LogDebug("[GitHubScraper] Skipping changelog search for {RepoKey} because cache indicates no changelog files exist.", repoKey); return null; } - // Otherwise, only try files that are known to exist changelogFiles = existingFiles; } else if (existingFiles is { Count: > 0 }) { - // Prioritize already discovered existing files even if repository is not fully resolved var prioritizedList = new List(existingFiles); foreach (var file in CandidateFiles) { @@ -135,7 +126,7 @@ public static bool IsRedirection(string notes) } var cleanVersion = version.TrimStart('v', 'V'); - // The version tag, then the default branch: HEAD is whichever branch the repository uses (master, main, develop) + // HEAD resolves to the repository's default branch, whatever it is named var refNames = new[] { $"v{cleanVersion}", cleanVersion, "HEAD" }; var attemptedFiles = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -143,7 +134,6 @@ public static bool IsRedirection(string notes) foreach (var fileName in changelogFiles) { - // Skip if we already know this file does not exist in this repo if (knowledge.IsMissing(fileName)) { continue; @@ -164,7 +154,6 @@ public static bool IsRedirection(string notes) { succeededFiles.Add(fileName); - // Since it succeeded, we know this file exists knowledge.AddExisting(fileName); var content = await response.Content.ReadAsStringAsync(); @@ -180,7 +169,7 @@ public static bool IsRedirection(string notes) } else if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { - // Missing on the default branch: the file is not in the repository + // Only a miss on the default branch means the file is not in the repository if (refName == "HEAD") { knowledge.MarkMissing(fileName); @@ -191,7 +180,6 @@ public static bool IsRedirection(string notes) } catch (OperationCanceledException) when (WorkCancellation.Token.IsCancellationRequested) { - // The work this lookup belongs to was cancelled: no point in trying the other files throw; } catch (Exception ex) @@ -205,7 +193,7 @@ public static bool IsRedirection(string notes) return null; } - // GitHub wiki pages are plain Markdown files of a separate repository (lodash keeps its changelog there) + // Wiki pages are plain Markdown files served from a separate repository private static readonly string[] WikiPages = { "Changelog", "CHANGELOG", "Change-Log", "Release-Notes", "Release-notes", "Changes", "History" }; private async Task FetchFromWikiAsync(string owner, string repo, string version, string? packageId) @@ -304,7 +292,7 @@ private async Task ResolveExistingFilesViaApiAsync(string owner, string repo, st } catch (GitHubRateLimitedException) { - // The candidate files are tried directly instead; raw file downloads do not count against the API limit + // Raw downloads do not count against the API limit, so candidates are tried directly _logger.LogDebug("[GitHubScraper] Rate limited, not listing repo contents of {RepoKey}", repoKey); } catch (Exception ex) @@ -340,14 +328,10 @@ private string ExtractVersionFromChangelog(string changelogText, string version, var lines = changelogText.Replace("\r\n", "\n").Replace("\r", "\n").Split('\n'); var cleanVer = version.TrimStart('v', 'V'); - // A heading names the version, optionally after a word ("Version", "Release", the package name), - // and ends the version with a space, a bracket, a link or the end of the line: - // ## v2.34.1 ## [2.34.1] - 2026-05-13 ## [10.0.0](https://...compare/v9.0.0...v10.0.0) (2024-06-07) - // ## Version 1.1.4 ## socket.io@4.8.3 ## react-dom 18.3.1 1.1.4 (2021-05-13) + underline + // Version heading, optionally after a word or package name, ending at a space, bracket or EOL var title = $@"(?:(?:version|release|v)\s*|[\w@/.-]+(?:\s+v?|@v?))?\[?[vV]?{Regex.Escape(cleanVer)}\]?(?=$|[\s(\[\]:,;])(?:.*)$"; var atxRegex = new Regex($@"^(?#+)\s+{title}", RegexOptions.IgnoreCase); - // A line that could be a Setext heading, e.g. "2.34.1 (2026-05-13)" or "Version 1.1.4" var setextTitleRegex = new Regex("^" + title, RegexOptions.IgnoreCase); var setextUnderlineRegex = new Regex(@"^[-=]{3,}$"); @@ -359,7 +343,6 @@ private string ExtractVersionFromChangelog(string changelogText, string version, { var line = HtmlTag().Replace(lines[i], string.Empty).Trim(); - // Check ATX first var match = atxRegex.Match(line); if (match.Success) { @@ -369,7 +352,6 @@ private string ExtractVersionFromChangelog(string changelogText, string version, break; } - // Check Setext if (setextTitleRegex.IsMatch(line) && i + 1 < lines.Length && setextUnderlineRegex.IsMatch(lines[i + 1].Trim())) { startIndex = i; @@ -382,8 +364,6 @@ private string ExtractVersionFromChangelog(string changelogText, string version, var sectionLines = new List(); - // If ATX heading of level N: terminate on any heading of level <= N (e.g. # or ##) - // If Setext heading: terminate on any ATX heading of level 1 or 2, or another Setext heading Regex terminatorRegex; if (isAtx) { @@ -391,18 +371,16 @@ private string ExtractVersionFromChangelog(string changelogText, string version, } else { - // Setext heading is equivalent to h1 or h2. Terminate on any h1 or h2 (e.g. # or ##) terminatorRegex = new Regex(@"^#{1,2}\s+\S+"); } - int startReadingFrom = isAtx ? startIndex + 1 : startIndex + 2; // Skip the underline for Setext + int startReadingFrom = isAtx ? startIndex + 1 : startIndex + 2; for (int i = startReadingFrom; i < lines.Length; i++) { var rawLine = lines[i]; var trimmedLine = rawLine.Trim(); - // Check if we hit another header if (isAtx) { if (terminatorRegex.IsMatch(trimmedLine)) @@ -410,13 +388,11 @@ private string ExtractVersionFromChangelog(string changelogText, string version, } else { - // For Setext, we also terminate if we find another Setext header if (terminatorRegex.IsMatch(trimmedLine)) break; if (i + 1 < lines.Length && setextUnderlineRegex.IsMatch(lines[i + 1].Trim())) { - // This line and next line form a Setext header, which terminates our section break; } } diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubFirstScrapingStrategy.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubFirstScrapingStrategy.cs index 91fb04a..a34fc76 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubFirstScrapingStrategy.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubFirstScrapingStrategy.cs @@ -5,7 +5,6 @@ namespace PkgDiff.Infrastructure.Services; // GitHub releases first, then the notes the registry itself publishes, when it publishes any. -// Used for registries whose notes only come from a repository and for NuGet packages with a known repository. public class GitHubFirstScrapingStrategy : IReleaseNotesScrapingStrategy { private readonly IGitHubScraperService _githubScraperService; @@ -36,7 +35,6 @@ public async Task GetReleaseNotesAsync( string? discoveredUrl = null; - // If no GitHub URL yet, try to discover from registry if (string.IsNullOrWhiteSpace(githubRepoUrl)) { _logger.LogDebug("[GitHubFirstStrategy] No GitHub URL, trying registry lookup for {PackageId} (Registry: {Registry})", packageId, registry.Id); @@ -56,12 +54,11 @@ public async Task GetReleaseNotesAsync( } } - // Try GitHub first if (!string.IsNullOrWhiteSpace(githubRepoUrl)) { _logger.LogDebug("[GitHubFirstStrategy] Trying GitHub releases for {PackageId} v{Version} from {GitHubUrl}", packageId, version, githubRepoUrl); - // A monorepo package keeps its changelog in its own folder; the URL tells the scraper where + // A monorepo package keeps its changelog in its own folder, which the URL names var lookupUrl = githubRepoUrl; if (GitHubUrls.SubPath(githubRepoUrl) is null) { @@ -81,7 +78,6 @@ public async Task GetReleaseNotesAsync( packageId, version, registry.PublishesReleaseNotes); } - // Fall back to the registry's own notes, when it publishes any if (registry.PublishesReleaseNotes) { _logger.LogDebug("[GitHubFirstStrategy] Trying {Registry} notes for {PackageId} v{Version}", registry.Id, packageId, version); @@ -91,7 +87,6 @@ public async Task GetReleaseNotesAsync( _logger.LogDebug("[GitHubFirstStrategy] {Registry} returned notes for {PackageId} v{Version} (length: {Length}): {Preview}...", registry.Id, packageId, version, ownNotes.Length, ownNotes[..Math.Min(150, ownNotes.Length)]); - // Check if the registry's notes are URL-dominated and try to resolve actual content var resolved = await _urlResolver.ResolveAsync(ownNotes, packageId, version, githubRepoUrl); if (resolved.WasResolved) { @@ -109,7 +104,6 @@ public async Task GetReleaseNotesAsync( _logger.LogWarning("[GitHubFirstStrategy] ALL SOURCES EMPTY for {PackageId} v{Version}", packageId, version); - // Persist failure to scrape logs for admin review var allReleases = !string.IsNullOrWhiteSpace(githubRepoUrl) ? await _githubScraperService.GetAllReleasesAsync(githubRepoUrl) : new Dictionary(); diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubMilestoneReader.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubMilestoneReader.cs index a2e9ee7..52bb9de 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubMilestoneReader.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubMilestoneReader.cs @@ -8,9 +8,8 @@ namespace PkgDiff.Infrastructure.Services; -// Release notes from GitHub milestones, for projects that publish neither releases nor a changelog: hapi's -// packages (joi, hapi, boom...) close one milestone per version, named after it, and its closed issues and -// pull requests are the changes of that version (https://joi.dev/resources/changelog is built from them) +// Release notes from closed GitHub milestones named after versions, for projects with neither +// releases nor a changelog public class GitHubMilestoneReader { private const int PerPage = 100; @@ -76,7 +75,7 @@ public GitHubMilestoneReader( return CompiledReleaseNotes.WithNotice(notes, $"{version.TrimStart('v', 'V')} milestone", milestoneUrl); } - // Closed milestones by version ("v18.3.0" and "18.3.0" alike); an empty map for a repository without any + // Closed milestones by version, with or without a "v"; an empty map for a repository without any private async Task> MilestonesAsync(string owner, string repo) { var repoKey = $"{owner}/{repo}".ToLowerInvariant(); @@ -87,8 +86,7 @@ private async Task> MilestonesAsync(string owner, string for (var page = 1; ; page++) { using var document = await GetJsonAsync($"{RepoApi(owner, repo)}/milestones?state=closed&per_page={PerPage}&page={page}"); - // A failed page leaves the map unknown rather than empty: it is asked again soon, but not - // once for every version of the package + // A failed page leaves the map unknown: retried soon, but not for every version if (document is null || document.RootElement.ValueKind != JsonValueKind.Array) { _cache.SetMilestones(repoKey, milestones, answered: false); diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitHandler.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitHandler.cs index 7e6b66a..592b1cc 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitHandler.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitHandler.cs @@ -3,18 +3,15 @@ namespace PkgDiff.Infrastructure.Services; -/// -/// Singleton service for tracking GitHub API rate limits from response headers -/// public class GitHubRateLimitHandler : IGitHubRateLimitHandler { - private const int SafetyBufferThreshold = 10; // Minimum requests before waiting for reset - private const int WarningThreshold = 100; // Threshold for logging warnings + private const int SafetyBufferThreshold = 10; + private const int WarningThreshold = 100; private readonly ILogger _logger; private readonly SemaphoreSlim _semaphore = new(1, 1); - private int _remaining = 5000; // Default for authenticated + private int _remaining = 5000; private int _limit = 5000; private DateTime? _resetTime; @@ -37,8 +34,7 @@ public async Task CanMakeRequestAsync() _resetTime = null; } - // Safety buffer: the caller stops instead of waiting, so background workers are not held - // for up to an hour and its work can be scheduled for the reset + // Stops instead of waiting so background workers are not held until the reset if (_remaining <= SafetyBufferThreshold && _resetTime.HasValue) { _logger.LogDebug( @@ -50,7 +46,6 @@ public async Task CanMakeRequestAsync() // Counted before the answer arrives, so requests running in parallel cannot spend the buffer _remaining--; - // Log warning when approaching limit if (_remaining < WarningThreshold && _remaining > SafetyBufferThreshold) { _logger.LogWarning( @@ -71,7 +66,6 @@ public async Task UpdateFromHeadersAsync(HttpResponseMessage response) await _semaphore.WaitAsync(); try { - // Parse x-ratelimit-remaining if (response.Headers.TryGetValues("x-ratelimit-remaining", out var remainingValues)) { var remainingStr = remainingValues.FirstOrDefault(); @@ -81,7 +75,6 @@ public async Task UpdateFromHeadersAsync(HttpResponseMessage response) } } - // Parse x-ratelimit-limit if (response.Headers.TryGetValues("x-ratelimit-limit", out var limitValues)) { var limitStr = limitValues.FirstOrDefault(); @@ -91,7 +84,6 @@ public async Task UpdateFromHeadersAsync(HttpResponseMessage response) } } - // Parse x-ratelimit-reset (Unix timestamp) if (response.Headers.TryGetValues("x-ratelimit-reset", out var resetValues)) { var resetStr = resetValues.FirstOrDefault(); diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitResponse.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitResponse.cs index 739f6e0..d8a85fd 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitResponse.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubRateLimitResponse.cs @@ -5,8 +5,7 @@ namespace PkgDiff.Infrastructure.Services; public static class GitHubRateLimitResponse { - // A 403 or 429 with no requests left is GitHub refusing because of the limit, not an answer about - // the repository: the caller must not take it for "no releases" or "no milestones" + // A rate-limit refusal says nothing about the repository: not "no releases" or "no milestones" public static void ThrowIfLimited(HttpResponseMessage response) { if (response.StatusCode is not (HttpStatusCode.Forbidden or HttpStatusCode.TooManyRequests)) diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseFileReader.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseFileReader.cs index fad54b5..4ed34da 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseFileReader.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseFileReader.cs @@ -6,9 +6,7 @@ namespace PkgDiff.Infrastructure.Services; -// Release notes kept as one file per version in a folder of the repository, e.g. Django's -// docs/releases/5.2.1.txt. The folder listings are read once per repository through the API, so a -// version costs one raw download, and a repository without such a folder nothing after the first look. +// Release notes kept as one file per version in a repository folder (listed once per repo) public class GitHubReleaseFileReader { private static readonly string[] Folders = @@ -80,7 +78,7 @@ public GitHubReleaseFileReader( } } - // "5.2.1" is 5.2.1.txt; "5.2" and "5.2.0" are each other's file too + // A trailing ".0" is optional: x.y and x.y.0 name the same file private static string? FileFor(Dictionary files, string version) { if (files.TryGetValue(version, out var path)) @@ -172,8 +170,7 @@ private async Task> ReleaseFilesAsync(string owner, s } } - // The page title of the file repeats the version: an RST title between two rules - // ("====\nDjango 5.2.1 release notes\n====") or a first Markdown heading + // The file's title repeats the version: an RST title between rules or a first Markdown heading private static string WithoutTitle(string text) { var lines = text.Replace("\r\n", "\n").Split('\n').ToList(); diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseHtmlScraper.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseHtmlScraper.cs index 99eab26..39dc72a 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseHtmlScraper.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleaseHtmlScraper.cs @@ -31,13 +31,11 @@ public async Task> GetAllReleasesAsync(string githubR _logger.LogDebug("[GitHubScraper] Fetching all releases via HTML: {ReleasesUrl}", baseReleasesUrl); - // OPTIMIZATION: Add timeout to prevent long waits due to rate limiting - // If timeout occurs, return partial results (user shouldn't wait too long) + // Bounded: on timeout the releases read so far are returned using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_htmlScrapingTimeoutSeconds)); try { - // Scrape first page with rate limiting using var document = await _rateLimiter.ExecuteAsync("github-scraping", () => _htmlDocumentLoader.LoadAsync(baseReleasesUrl)); if (document == null) { @@ -47,25 +45,20 @@ public async Task> GetAllReleasesAsync(string githubR _logger.LogDebug("[GitHubScraper] Page title: {Title}", document.Title); - // Extract releases from first page GitHubReleasePage.ExtractReleases(document, releases, _logger); - // Check for pagination var totalPages = GitHubReleasePage.GetTotalPages(document); _logger.LogDebug("[GitHubScraper] Total pages detected: {TotalPages}", totalPages); - // Scrape all pages (rate limiter will manage speed) var maxPagesToScrape = totalPages; for (int page = 2; page <= maxPagesToScrape; page++) { - // Check for timeout before each page fetch if (cts.Token.IsCancellationRequested) { _logger.LogWarning("[GitHubScraper] TIMEOUT after {Timeout}s - Returning partial results ({ReleasesCount} releases from {PagesScraped}/{TotalPages} pages)", _htmlScrapingTimeoutSeconds, releases.Count, page - 1, totalPages); - // Return partial results to user (don't make them wait) return releases; } @@ -74,7 +67,6 @@ public async Task> GetAllReleasesAsync(string githubR try { - // Use rate limiter - it will automatically wait if needed using var pageDocument = await _rateLimiter.ExecuteAsync("github-scraping", () => _htmlDocumentLoader.LoadAsync(pageUrl)); if (pageDocument != null) { @@ -106,13 +98,11 @@ public async Task> GetAllReleasesAsync(string githubR _logger.LogDebug("[GitHubScraper] Using LAZY HTML scraping for version {Version}", cleanVersion); - // Add timeout for lazy scraping too (half of full scraping timeout) var timeoutSeconds = _htmlScrapingTimeoutSeconds / 2; using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); try { - // Fetch first page using var document = await _rateLimiter.ExecuteAsync("github-scraping", () => _htmlDocumentLoader.LoadAsync(baseReleasesUrl)); if (document == null) { @@ -120,11 +110,9 @@ public async Task> GetAllReleasesAsync(string githubR return null; } - // Extract releases from first page and check if our version is there var releases = new Dictionary(); GitHubReleasePage.ExtractReleases(document, releases, _logger); - // Check if we found it on first page var result = TryFindVersion(releases, cleanVersion); if (result != null) { @@ -132,15 +120,12 @@ public async Task> GetAllReleasesAsync(string githubR return result; } - // Check pagination var totalPages = GitHubReleasePage.GetTotalPages(document); _logger.LogDebug("[GitHubScraper] Version not found on page 1, checking up to {TotalPages} more pages", totalPages - 1); - // Fetch additional pages until we find the version or timeout for (int page = 2; page <= totalPages; page++) { - // Check for timeout before each page fetch if (cts.Token.IsCancellationRequested) { _logger.LogWarning("[GitHubScraper] TIMEOUT after {Timeout}s while searching for version {Version} - Version not found in first {Pages} pages", @@ -152,14 +137,13 @@ public async Task> GetAllReleasesAsync(string githubR try { - releases.Clear(); // Clear previous page releases + releases.Clear(); using var pageDocument = await _rateLimiter.ExecuteAsync("github-scraping", () => _htmlDocumentLoader.LoadAsync(pageUrl)); if (pageDocument != null) { GitHubReleasePage.ExtractReleases(pageDocument, releases, _logger); - // Check if we found it on this page result = TryFindVersion(releases, cleanVersion); if (result != null) { @@ -187,19 +171,16 @@ public async Task> GetAllReleasesAsync(string githubR private static string? TryFindVersion(Dictionary releases, string targetVersion) { - // Try exact match if (releases.TryGetValue(targetVersion, out var notes)) { return notes; } - // Try with 'v' prefix if (releases.TryGetValue($"v{targetVersion}", out notes)) { return notes; } - // Try fuzzy match var fuzzyMatch = releases.FirstOrDefault(kvp => kvp.Key.Replace(".", "").Replace("-", "").Replace("_", "") .Equals(targetVersion.Replace(".", "").Replace("-", "").Replace("_", ""), diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleasePage.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleasePage.cs index abc4b78..9d0b28e 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubReleasePage.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubReleasePage.cs @@ -7,20 +7,17 @@ namespace PkgDiff.Infrastructure.Services; // Reads a GitHub releases page: the releases on it, its page count, and the notes as Markdown public static class GitHubReleasePage { - // Regex to extract semantic version from release titles like "Aspire 13.1.1", "v9.5.0 Release", "Package Name 1.2.3-preview.1" private static readonly Regex SemVerRegex = new( @"(?\d+\.\d+\.\d+(?:[-\.][a-zA-Z0-9\.]+)?)", RegexOptions.Compiled); public static void ExtractReleases(IDocument document, Dictionary releases, ILogger logger) { - // Find all release sections var releaseSections = document.QuerySelectorAll("section[aria-labelledby]"); logger.LogDebug("[GitHubScraper] Found {Count} release sections on current page", releaseSections.Length); foreach (var section in releaseSections) { - // Extract version from the heading var versionElement = section.QuerySelector(".f1.text-bold a, h2 a"); if (versionElement == null) { @@ -30,20 +27,17 @@ public static void ExtractReleases(IDocument document, Dictionary "releases:" + repositoryUrl; } -// Which changelog files a repository has, as far as earlier lookups could tell. Shared between -// requests, so every access is locked. +// Which changelog files a repository has, as far as lookups could tell; shared, so locked public sealed class RepoChangelogKnowledge { public const int FailuresBeforeMissing = 3; diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs index 65af6ab..9028542 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs @@ -42,20 +42,17 @@ public GitHubScraperService( _htmlDocumentLoader = htmlDocumentLoader; _cache = cache; - // Load GitHub settings from configuration var githubSettings = configuration.GetSection("GitHubSettings"); var personalAccessToken = githubSettings.GetValue("PersonalAccessToken"); _enableAuthentication = githubSettings.GetValue("EnableAuthentication", true); - _useGitHubApi = githubSettings.GetValue("UseGitHubApi", true); // New: prefer API over HTML scraping - var htmlScrapingTimeoutSeconds = githubSettings.GetValue("HtmlScrapingTimeoutSeconds", 30); // Default: 30s timeout + _useGitHubApi = githubSettings.GetValue("UseGitHubApi", true); + var htmlScrapingTimeoutSeconds = githubSettings.GetValue("HtmlScrapingTimeoutSeconds", 30); var userAgent = githubSettings.GetValue("UserAgent", "PkgDiff/1.0"); - // Create dedicated HttpClient for GitHub API calls using factory _httpClient = httpClientFactory.CreateClient(HttpClientNames.GitHubApi); _httpClient.DefaultRequestHeaders.Add("User-Agent", userAgent); - _httpClient.DefaultRequestHeaders.Add("Accept", "application/vnd.github+json"); // GitHub API v3 + _httpClient.DefaultRequestHeaders.Add("Accept", "application/vnd.github+json"); - // Add authentication if token is provided and enabled if (_enableAuthentication && !string.IsNullOrWhiteSpace(personalAccessToken)) { _httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {personalAccessToken}"); @@ -66,7 +63,6 @@ public GitHubScraperService( _releaseFiles = new GitHubReleaseFileReader(_httpClient, rateLimiter, rateLimitHandler, cache, _useGitHubApi && _enableAuthentication, logger); _milestones = new GitHubMilestoneReader(_httpClient, rateLimiter, rateLimitHandler, cache, _useGitHubApi && _enableAuthentication, logger); - // Polly retry policy: 3 attempts with exponential backoff, for transient failures only _retryPolicy = Policy .Handle(TransientHttp.IsTransient) .WaitAndRetryAsync(3, retryAttempt => @@ -80,7 +76,6 @@ public GitHubScraperService( var url = $"https://www.nuget.org/packages/{PackageUrlEncoder.EscapeSegment(packageId)}"; _logger.LogDebug("[GitHubScraper] Extracting GitHub repo URL for {PackageId} from {Url}", packageId, url); - // Use rate limiter for NuGet scraping using var document = await _rateLimiter.ExecuteAsync("nuget-scraping", () => _htmlDocumentLoader.LoadAsync(url)); if (document == null) { @@ -88,7 +83,6 @@ public GitHubScraperService( return null; } - // Find GitHub repository link var links = document.QuerySelectorAll("aside a[href*='github.com']"); var githubLink = links.FirstOrDefault(); @@ -121,7 +115,6 @@ public async Task> GetAllReleasesAsync(string githubR return cachedReleases; } - // Use GitHub API if enabled and authenticated, otherwise fall back to HTML scraping Dictionary releases; bool answered; if (_useGitHubApi && _enableAuthentication) @@ -134,7 +127,7 @@ public async Task> GetAllReleasesAsync(string githubR answered = releases.Count > 0; } - // Kept even when empty: the caller asks again for every version of the package + // Cached even when empty: the caller asks again for every version _cache.SetReleases(cleanUrl, releases, answered); _logger.LogDebug("[GitHubScraper] Cached {Count} releases for {RepoUrl} (answered: {Answered})", releases.Count, cleanUrl, answered); @@ -156,7 +149,7 @@ public async Task> GetAllReleasesAsync(string githubR } int page = 1; - int perPage = 100; // GitHub API max per page + int perPage = 100; bool hasMorePages = true; bool answered = false; @@ -168,17 +161,14 @@ public async Task> GetAllReleasesAsync(string githubR try { - // Check rate limit before making request await _rateLimitHandler.CanMakeRequestAsync(); - // Use github-api rate limiter (5000 req/hour) using var response = await _rateLimiter.ExecuteAsync("github-api", async () => { _logger.LogDebug("[GitHubScraper] API Request: GET {ApiUrl}", apiUrl); return await _httpClient.GetAsync(apiUrl); }); - // Update rate limit from response headers await _rateLimitHandler.UpdateFromHeadersAsync(response); if (!response.IsSuccessStatusCode) @@ -186,7 +176,7 @@ public async Task> GetAllReleasesAsync(string githubR GitHubRateLimitResponse.ThrowIfLimited(response); _logger.LogWarning("[GitHubScraper] API returned {StatusCode} for {Url}", response.StatusCode, apiUrl); - // A repository that does not exist (any more) will not have releases a minute later + // A missing repository will not have releases a minute later answered = response.StatusCode == System.Net.HttpStatusCode.NotFound; break; } @@ -211,8 +201,6 @@ public async Task> GetAllReleasesAsync(string githubR { var notes = GitHubReleasePage.CleanReleaseNotes(body); - // Every tag under its own name ("socket.io@4.8.3"); a plain version tag also - // without its "v", as it always was releases.TryAdd(tagName, notes); if (PlainVersionTag().IsMatch(tagName)) releases.TryAdd(tagName.TrimStart('v', 'V'), notes); @@ -220,7 +208,6 @@ public async Task> GetAllReleasesAsync(string githubR } } - // Check if there are more pages (GitHub returns less than perPage if it's the last page) hasMorePages = releasesArray.GetArrayLength() == perPage; answered = !hasMorePages; page++; @@ -256,40 +243,33 @@ public async Task> GetAllReleasesAsync(string githubR string? notes = null; - // OPTIMIZATION: Use lazy loading for HTML scraping - // Instead of fetching ALL pages, fetch page-by-page until we find the version if (!_useGitHubApi || !_enableAuthentication) { notes = await _htmlScraper.GetReleaseNotesForVersionAsync(githubRepoUrl, version); } else { - // For API mode, fetch all releases (it's fast with API) var allReleases = await GetAllReleasesAsync(GitHubUrls.Clean(githubRepoUrl)); - // Clean the version string for comparison var cleanVersion = version.TrimStart('v', 'V'); - // A monorepo tags each package on its own: its name decides which tag is this package's + // Monorepos tag each package separately; the package name picks its tag var packageTag = TagCandidates(packageId, cleanVersion).FirstOrDefault(allReleases.ContainsKey); if (packageTag is not null) { notes = allReleases[packageTag]; _logger.LogDebug("[GitHubScraper] Package tag {Tag} found for version {Version}", packageTag, cleanVersion); } - // Try exact match first else if (allReleases.TryGetValue(cleanVersion, out notes)) { _logger.LogDebug("[GitHubScraper] Exact match found for version {Version}", cleanVersion); } - // Try with 'v' prefix else if (allReleases.TryGetValue($"v{cleanVersion}", out notes)) { _logger.LogDebug("[GitHubScraper] Match found with 'v' prefix for version v{Version}", cleanVersion); } else { - // Try fuzzy match (in case of minor formatting differences) var fuzzyMatch = allReleases.FirstOrDefault(kvp => kvp.Key.Replace(".", "").Replace("-", "").Replace("_", "") .Equals(cleanVersion.Replace(".", "").Replace("-", "").Replace("_", ""), @@ -308,7 +288,6 @@ public async Task> GetAllReleasesAsync(string githubR } } - // Check if notes are empty or redirection if (string.IsNullOrWhiteSpace(notes) || GitHubChangelogReader.IsRedirection(notes)) { _logger.LogDebug("[GitHubScraper] Notes are empty or redirection for version {Version}. Attempting to fetch from repository changelog files.", version); @@ -319,8 +298,7 @@ public async Task> GetAllReleasesAsync(string githubR } else { - // No release and no changelog: one file per version (Django's docs/releases), else the - // project may track its versions as milestones (hapi, joi) + // No release and no changelog: try per-version release files, then milestones var cleanUrl = GitHubUrls.Clean(githubRepoUrl); var otherNotes = await _releaseFiles.FetchAsync(cleanUrl, version) ?? await _milestones.FetchAsync(cleanUrl, version); @@ -335,7 +313,7 @@ public async Task> GetAllReleasesAsync(string githubR }); } - // Tag spellings of monorepos and of projects that name their tags after the package, most specific first + // Tag spellings of monorepos and package-named tags, most specific first private static IEnumerable TagCandidates(string? packageId, string version) { if (string.IsNullOrWhiteSpace(packageId)) @@ -360,7 +338,6 @@ private static IEnumerable TagCandidates(string? packageId, string versi yield return $"release/{version}"; } - // "1.2.3", "v1.2.3", "V1.2.3-beta.1": a tag that is nothing but a version [System.Text.RegularExpressions.GeneratedRegex(@"^[vV]?\d")] private static partial System.Text.RegularExpressions.Regex PlainVersionTag(); } diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubUrls.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubUrls.cs index b0aec76..bc12b52 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubUrls.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubUrls.cs @@ -12,7 +12,6 @@ public static string Clean(string url) url = url.TrimEnd('/'); - // Remove common GitHub paths to get base repository URL var segments = new[] { "/releases", "/issues", "/wiki", "/pulls", "/actions", "/tree", "/blob", "/commit" }; foreach (var segment in segments) { @@ -27,8 +26,7 @@ public static string Clean(string url) return url; } - // The folder a repository URL points into: "packages/core" for - // https://github.com/o/r/tree/main/packages/core; null for the repository itself or a file + // The folder a repository URL points into; null for the repository itself or a file public static string? SubPath(string? url) { if (string.IsNullOrWhiteSpace(url)) @@ -38,16 +36,14 @@ public static string Clean(string url) if (!match.Success) return null; var path = Uri.UnescapeDataString(match.Groups["path"].Value).Trim('/'); - // A link to a file (packages/core/README.md) points into the folder that holds it + // A link to a file points into the folder that holds it var lastSlash = path.LastIndexOf('/'); if (System.Text.RegularExpressions.Regex.IsMatch(path[(lastSlash + 1)..], @"\.(md|markdown|rst|txt|adoc)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) path = lastSlash < 0 ? string.Empty : path[..lastSlash]; return path.Length == 0 || path.Contains("..") ? null : path; } - // The https URL of a repository as a package manifest names it: "github:o/r", the bare "o/r" npm - // takes for GitHub, "git@github.com:o/r.git", "ssh://git@github.com/o/r", "git+https://github.com/o/r.git". - // Anything else comes back without its git decorations. + // The https URL of a repository in any spelling a manifest uses (shorthand, ssh, git+https) public static string? FromRepositoryField(string? repository) { if (string.IsNullOrWhiteSpace(repository)) @@ -71,8 +67,7 @@ public static string Clean(string url) return TrimGitSuffix(url.TrimEnd('/')); } - // True for an address on github.com in any spelling a manifest uses; a host that merely contains - // "github.com" (github.com.example.org) is not GitHub + // True for github.com in any manifest spelling; a host merely containing it is not GitHub public static bool IsGitHub(string? url) { var normalized = FromRepositoryField(url); @@ -81,7 +76,7 @@ public static bool IsGitHub(string? url) || uri.Host.Equals("www.github.com", StringComparison.OrdinalIgnoreCase)); } - // Only the ending: "bar.github.io" keeps its ".git" + // Only the ending is trimmed; a ".git" inside a host name stays private static string TrimGitSuffix(string url) => url.EndsWith(".git", StringComparison.OrdinalIgnoreCase) ? url[..^4] : url; diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubUserValidatorService.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubUserValidatorService.cs index 080dd22..114c865 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubUserValidatorService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubUserValidatorService.cs @@ -7,11 +7,6 @@ namespace PkgDiff.Infrastructure.Services; -/// -/// Validates GitHub usernames by calling the GitHub API users endpoint. -/// Filters out Organizations (only real User accounts are valid contributors). -/// Uses the same authenticated HTTP client as GitHubScraperService. -/// public class GitHubUserValidatorService { private static readonly TimeSpan SharedLifetime = TimeSpan.FromHours(6); @@ -22,7 +17,6 @@ public class GitHubUserValidatorService private readonly IGitHubRateLimitHandler? _rateLimitHandler; private readonly IMemoryCache? _sharedCache; - // In-process cache so repeated mentions of same username don't hit API twice private readonly Dictionary _cache = new(StringComparer.OrdinalIgnoreCase); public GitHubUserValidatorService( @@ -51,10 +45,6 @@ public GitHubUserValidatorService( _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer {token}"); } - /// - /// Returns true only if the username is a real GitHub User (not an Organization or Bot). - /// Results are cached in-memory for the lifetime of the scoped service. - /// public async Task IsValidGitHubUserAsync(string username) { if (_cache.TryGetValue(username, out var cached)) @@ -65,7 +55,7 @@ public async Task IsValidGitHubUserAsync(string username) try { - // Through the same budget as every other GitHub call: mentions are written by package authors + // Shares the GitHub budget: mentions are written by package authors var url = $"https://api.github.com/users/{PackageUrlEncoder.EscapeSegment(username)}"; if (_rateLimitHandler != null) await _rateLimitHandler.CanMakeRequestAsync(); @@ -83,12 +73,11 @@ public async Task IsValidGitHubUserAsync(string username) if (!response.IsSuccessStatusCode) { - // Rate limited or failing: not known yet, so not accepted now and asked again on a later scrape + // Rate limited or failing: not accepted now, asked again on a later scrape _logger.LogDebug("[GitHubValidator] @{Username} could not be checked ({Status})", username, (int)response.StatusCode); return false; } - // Parse the response body to check the account type var body = await response.Content.ReadAsStringAsync(); using var doc = JsonDocument.Parse(body); @@ -96,7 +85,6 @@ public async Task IsValidGitHubUserAsync(string username) ? typeProp.GetString() : null; - // Only accept real User accounts — reject Organizations and Bots var isUser = string.Equals(accountType, "User", StringComparison.OrdinalIgnoreCase); Remember(username, sharedKey, isUser); @@ -107,7 +95,7 @@ public async Task IsValidGitHubUserAsync(string username) } catch (Exception ex) { - // Not accepted: a name that cannot be checked is not shown as a contributor; a later scrape asks again + // A name that cannot be checked is not shown; a later scrape asks again _logger.LogWarning(ex, "[GitHubValidator] Could not validate @{Username}", username); return false; } diff --git a/src/backend/PkgDiff.Infrastructure/Services/HealthCheckService.cs b/src/backend/PkgDiff.Infrastructure/Services/HealthCheckService.cs index 0065c82..d1f7f8c 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/HealthCheckService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/HealthCheckService.cs @@ -43,11 +43,9 @@ public async Task GetDetailedHealthAsync() { var checkedAt = DateTime.UtcNow; - // The settings come from the same DbContext as the database check, which allows one - // operation at a time: they are read before the checks run in parallel + // Read before the parallel checks: the shared DbContext allows one operation at a time var providerSettings = await ReadProviderSettingsAsync(); - // Run all checks in parallel var dbTask = CheckDatabaseHealthAsync(); var nugetTask = CheckNuGetHealthAsync(); var githubTask = CheckGitHubHealthAsync(); @@ -62,7 +60,6 @@ public async Task GetDetailedHealthAsync() var githubHealth = await githubTask; var hangfireHealth = await hangfireTask; - // Determine overall status var overallStatus = DetermineOverallStatus(dbHealth, nugetHealth, githubHealth, hangfireHealth); return new HealthDetails( @@ -149,15 +146,11 @@ private async Task CheckHangfireHealthAsync() { var monitoringApi = JobStorage.Current.GetMonitoringApi(); - // Get server count var servers = monitoringApi.Servers(); var activeServers = servers.Count; - // Get job stats var stats = monitoringApi.GetStatistics(); - // Determine health: considered healthy if at least 1 server is active - // degraded if failed jobs > 50 (configurable threshold) var isHealthy = activeServers > 0; string? errorMessage = null; @@ -167,7 +160,6 @@ private async Task CheckHangfireHealthAsync() } else if (stats.Failed > 50) { - // Don't mark as unhealthy, just note it (status calculation will handle degraded) errorMessage = $"High number of failed jobs: {stats.Failed}"; } @@ -193,7 +185,6 @@ private async Task CheckDatabaseHealthAsync() var sw = Stopwatch.StartNew(); try { - // Simple connectivity check await _dbContext.Database.ExecuteSqlRawAsync("SELECT 1"); sw.Stop(); @@ -240,7 +231,6 @@ private async Task CheckGitHubHealthAsync() var sw = Stopwatch.StartNew(); try { - // Load GitHub settings var githubSettings = _configuration.GetSection("GitHubSettings"); var personalAccessToken = githubSettings.GetValue("PersonalAccessToken"); var enableAuthentication = githubSettings.GetValue("EnableAuthentication", true); @@ -249,7 +239,6 @@ private async Task CheckGitHubHealthAsync() var request = new HttpRequestMessage(HttpMethod.Get, "https://api.github.com/rate_limit"); request.Headers.Add("User-Agent", userAgent); - // Add authentication if token is provided and enabled var isAuthenticated = false; if (enableAuthentication && !string.IsNullOrWhiteSpace(personalAccessToken)) { @@ -299,7 +288,6 @@ private async Task CheckGitHubHealthAsync() } } - // Example: {"resources":{"core":{"limit":5000,"remaining":4990,"reset":1790000000,"used":10}}} private static (int? Remaining, DateTime? ResetAt) ParseGitHubRateLimit(string jsonContent) { try @@ -319,7 +307,6 @@ private static (int? Remaining, DateTime? ResetAt) ParseGitHubRateLimit(string j } catch (Exception ex) when (ex is JsonException or InvalidOperationException) { - // The response does not have the expected shape return (null, null); } } @@ -330,19 +317,15 @@ private string DetermineOverallStatus( ExternalServiceHealth github, HangfireHealth hangfire) { - // Database is critical if (!db.IsHealthy) return "Unhealthy"; - // Hangfire server is critical for background jobs if (!hangfire.IsHealthy) return "Unhealthy"; - // Check for degraded conditions if (hangfire.FailedJobs > 50) return "Degraded"; - // External services are less critical var externalServicesDown = (!nuget.IsAvailable ? 1 : 0) + (!github.IsAvailable ? 1 : 0); return externalServicesDown switch diff --git a/src/backend/PkgDiff.Infrastructure/Services/ListedVersionsRegistryClient.cs b/src/backend/PkgDiff.Infrastructure/Services/ListedVersionsRegistryClient.cs index df23217..5bb5066 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ListedVersionsRegistryClient.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ListedVersionsRegistryClient.cs @@ -54,8 +54,7 @@ public async Task> GetVersionsAsync(string package public abstract Task GetPublishedDateAsync(string packageId, string version); - // The version list of the package page is what the background scrape walks, so it is cut by the - // same exclude patterns as the versions of an analysis + // The background scrape walks this list, so it gets the same exclude patterns as an analysis public async Task GetMetadataAsync(string packageId) { var metadata = await FetchMetadataAsync(packageId); diff --git a/src/backend/PkgDiff.Infrastructure/Services/ManifestParser.cs b/src/backend/PkgDiff.Infrastructure/Services/ManifestParser.cs index 0ece45e..db209e7 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ManifestParser.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ManifestParser.cs @@ -11,9 +11,8 @@ namespace PkgDiff.Infrastructure.Services; -// Reads the packages out of a dependency file a visitor pasted. The type is recognised from the -// content, so the file name is only a hint. Every requirement is reduced to one version to check: -// the exact one when the file pins it, otherwise the lowest version the requirement allows. +// Reads packages from a pasted dependency file; the type is detected from the content. +// Each requirement becomes one version: the pinned one, else the lowest it allows. public partial class ManifestParser : IManifestParser { private const string Dev = "dev"; @@ -38,7 +37,7 @@ public ManifestParseResult Parse(string registry, string content, string? fileNa return Finish(result); } - // One entry per package (the first one wins), at most the audit limit + // One entry per package (the first one wins), capped at the audit limit private static ManifestParseResult Finish(ManifestParseResult result) { var seen = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -64,8 +63,6 @@ private static ManifestParseResult Finish(ManifestParseResult result) return result; } - // ---------------------------------------------------------------- npm - private static ManifestParseResult ParseNpm(string content) { var root = ParseJson(content, "a package.json or package-lock.json"); @@ -165,8 +162,6 @@ private static AuditDependency FromNpmRange(string name, string? requirement, st }; } - // ---------------------------------------------------------------- NuGet - private static ManifestParseResult ParseNuGet(string content) { if (content.StartsWith('{')) @@ -175,7 +170,7 @@ private static ManifestParseResult ParseNuGet(string content) XDocument document; try { - // No DTD at all: a dependency file needs none, and one could expand entities ("billion laughs") + // No DTD: a dependency file needs none, and one could expand entities (billion laughs) using var reader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, @@ -296,8 +291,6 @@ private static AuditDependency FromNuGetRange(string id, string? requirement, st }; } - // ---------------------------------------------------------------- PyPI - private static ManifestParseResult ParsePyPI(string content, string? fileName) { if (content.StartsWith('{')) @@ -413,7 +406,6 @@ private static void AddPep508(TomlArray list, string? scope, ManifestParseResult } } - // Poetry and Pipfile tables: name = "^1.2" | "*" | { version = "..." , ... } private static void ReadPoetryStyle(TomlTable parent, string key, string? scope, ManifestParseResult result) { if (!parent.TryGetValue(key, out var value) || value is not TomlTable dependencies) @@ -456,7 +448,6 @@ private static ManifestParseResult ParsePipfileLock(JsonElement root) return result; } - // "requests[socks]>=2.31,<3 ; python_version >= '3.8'" private static AuditDependency? FromPep508(string line, string? scope) { var withoutMarker = line.Split(';')[0].Trim(); @@ -494,8 +485,6 @@ private static AuditDependency FromPythonConstraint(string name, string? require }; } - // ---------------------------------------------------------------- pub.dev - private static ManifestParseResult ParsePub(string content) { YamlMappingNode root; @@ -560,9 +549,7 @@ private static ManifestParseResult ParsePub(string content) return pubspec; } - // YamlStream builds its nodes recursively: a document nested a few thousand levels deep ("a: [[[[...") - // overflows the stack, and a stack overflow ends the whole process, where no catch can stop it. The - // event parser is not recursive, so the depth is measured with it first. + // YamlStream recurses and a stack overflow kills the process, so depth is measured first private const int MaxYamlDepth = 64; private static void RejectDeepYaml(string content) @@ -600,8 +587,6 @@ private static AuditDependency FromPubConstraint(string name, string? requiremen }; } - // ---------------------------------------------------------------- helpers - private static JsonElement ParseJson(string content, string expected) { try diff --git a/src/backend/PkgDiff.Infrastructure/Services/MarkdownComplexityGuard.cs b/src/backend/PkgDiff.Infrastructure/Services/MarkdownComplexityGuard.cs index 2545a37..ea31c33 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/MarkdownComplexityGuard.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/MarkdownComplexityGuard.cs @@ -1,19 +1,13 @@ namespace PkgDiff.Infrastructure.Services; -// Markdig, HtmlSanitizer and AngleSharp walk nested structures recursively, and a stack overflow cannot be -// caught: it ends the whole process. Nested raw HTML overflows a 1.5 MB thread stack (the Linux default) in the -// sanitizer's AngleSharp Node.Normalize at about 5,400 levels. This scan is linear, never recurses, and runs -// before either library sees the text. It over-estimates rather than under-estimates, so hostile input is -// rejected long before it gets anywhere near the crash depth. +// Rejects input nested deep enough to overflow the recursive parsers' stack, which ends the +// process. Linear scan that over-estimates depth so hostile input is rejected early. internal static class MarkdownComplexityGuard { - // 128 KiB of UTF-16 text. GitHub caps a release body at 125,000 characters and blog posts are a few tens of - // KiB. The sanitizer's cost grows faster than linearly with sibling count, so this also keeps the worst - // hostile note to about two seconds of CPU. + // Also bounds the sanitizer's superlinear cost on hostile notes internal const int MaxLength = 128 * 1024; - // Real notes stay below about 10 levels. The HTML parser can build at most about twice the counted depth - // (implied table sections and rows), which is still about 40 times below the measured crash. + // The HTML parser builds at most about twice the counted depth, far below the crash depth internal const int MaxDepth = 64; internal enum Limit @@ -34,13 +28,12 @@ internal enum Limit private static readonly HashSet VoidTags = new(StringComparer.Ordinal) { "br", "img", "hr" }; - // Closed by the HTML parser's adoption agency, which can keep or re-create them, so only an exact match closes them + // The adoption agency may keep or re-create these, so only an exact match closes them private static readonly HashSet FormattingTags = new(StringComparer.Ordinal) { "a", "b", "strong", "i", "em", "code", "s" }; - // Elements the HTML parser closes implicitly when an enclosing element ends private static readonly HashSet ImpliedEndTags = new(StringComparer.Ordinal) { "p", "li", "td", "th", "tr", "thead", "tbody", "tfoot" @@ -276,8 +269,7 @@ private static (bool CanOpen, bool CanClose) Flanking(char delimiter, char befor return (leftFlanking, rightFlanking); } - // Tracks open raw HTML elements the way the HTML parser does, but only closes an element when the parser - // certainly would, so the count never falls below the depth the parser builds + // Mirrors the parser's open elements but closes only when it certainly would (never undercounts) private static Limit CheckHtml(ReadOnlySpan line, List openElements, out int measured) { measured = 0; diff --git a/src/backend/PkgDiff.Infrastructure/Services/NpmService.cs b/src/backend/PkgDiff.Infrastructure/Services/NpmService.cs index eed5fda..94332df 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/NpmService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/NpmService.cs @@ -110,8 +110,7 @@ public async Task GetPackageMetadataAsync(string packageId) { try { - // Read as a stream: the packument of a package with thousands of versions (@prisma/client, - // 68 MB) is larger than the client's buffer cap, while the fields kept from it are small + // Streamed: large packuments exceed the client's buffer cap and only small fields are kept using var response = await _httpClient.GetAsync( PackageUrlEncoder.EscapeNpmPackagePath(packageId), HttpCompletionOption.ResponseHeadersRead); if (!response.IsSuccessStatusCode) @@ -130,7 +129,6 @@ public async Task GetPackageMetadataAsync(string packageId) } } - // Still a cap, only a higher one than the buffered read had private const long MaxPackumentBytes = 256L * 1024 * 1024; private static bool IsPrerelease(string version) => @@ -144,11 +142,6 @@ private static bool IsGitHubUrl(string? url) return GitHubUrls.IsGitHub(url); } - // ==================== HELPER METHODS ==================== - - /// - /// Cleans npm repository URL format (removes git+, git://, .git suffixes). - /// private static string? CleanRepositoryUrl(string? repositoryUrl) { if (string.IsNullOrEmpty(repositoryUrl)) @@ -157,9 +150,6 @@ private static bool IsGitHubUrl(string? url) return GitHubUrls.FromRepositoryField(repositoryUrl); } - /// - /// Maps npm package data to PackageMetadata DTO. - /// private PackageMetadata MapToPackageMetadata(string packageId, NpmPackageData packageData) { var latestVersion = packageData.DistTags?.Latest ?? packageData.Versions?.Keys.LastOrDefault(); @@ -167,7 +157,6 @@ private PackageMetadata MapToPackageMetadata(string packageId, NpmPackageData pa ? packageData.Versions[latestVersion] : null; - // Get repository/homepage URL string? repositoryUrl = latestVersionData?.Repository?.Url; repositoryUrl = CleanRepositoryUrl(repositoryUrl); // A monorepo names the folder of this package, where its changelog usually is @@ -182,24 +171,23 @@ private PackageMetadata MapToPackageMetadata(string packageId, NpmPackageData pa LatestVersion = latestVersion ?? "unknown", Description = latestVersionData?.Description ?? packageData.Description ?? "", Authors = latestVersionData?.Author?.Name ?? packageData.Author?.Name ?? "Unknown", - IconUrl = null, // npm doesn't provide icon URLs in registry + IconUrl = null, ProjectUrl = homepage ?? repositoryUrl, GitHubRepositoryUrl = IsGitHubUrl(repositoryUrl) ? repositoryUrl : null, SourceDirectory = IsGitHubUrl(repositoryUrl) ? directory : null, Versions = packageData.Versions? - .Where(v => !IsPrerelease(v.Key)) // Filter out prerelease versions by default + .Where(v => !IsPrerelease(v.Key)) .Select(v => new VersionSummary { Version = v.Key, PublishedDate = packageData.Time?.ContainsKey(v.Key) == true ? packageData.Time[v.Key] : null, - IsPrerelease = false, // Already filtered + IsPrerelease = false, }) .OrderByDescending(v => TryParseVersion(v.Version)) .ToList() ?? new List() }; } - // NPM API response models private class NpmPackageData { [JsonPropertyName("name")] @@ -339,9 +327,7 @@ public override void Write(Utf8JsonWriter writer, NpmRepository value, JsonSeria => JsonSerializer.Serialize(writer, value, new JsonSerializerOptions()); } - // A string field an author wrote as a list (fs-extra 0.0.1: "homepage": [...]) or a number: the first - // string of a list, the text of a scalar, nothing for anything else. One odd old version must not make - // the whole package unreadable. + // Lenient for odd old versions: first string of a list, text of a scalar, else null private class LooseStringConverter : JsonConverter { public override bool HandleNull => true; diff --git a/src/backend/PkgDiff.Infrastructure/Services/NuGetFirstScrapingStrategy.cs b/src/backend/PkgDiff.Infrastructure/Services/NuGetFirstScrapingStrategy.cs index f940a6b..0382f3e 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/NuGetFirstScrapingStrategy.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/NuGetFirstScrapingStrategy.cs @@ -3,7 +3,7 @@ namespace PkgDiff.Infrastructure.Services; -// The NuGet package page first, then GitHub releases. Used for NuGet packages without a known repository. +// The NuGet package page first, then GitHub releases public class NuGetFirstScrapingStrategy : IReleaseNotesScrapingStrategy { private readonly IScraperService _scraperService; @@ -34,7 +34,6 @@ public async Task GetReleaseNotesAsync( string? discoveredUrl = null; - // Try NuGet first _logger.LogDebug("[NuGetFirstStrategy] Trying NuGet scraping for {PackageId} v{Version}", packageId, version); var releaseNotes = await _scraperService.GetReleaseNotesAsync(packageId, version); if (!string.IsNullOrWhiteSpace(releaseNotes)) @@ -42,7 +41,6 @@ public async Task GetReleaseNotesAsync( _logger.LogDebug("[NuGetFirstStrategy] NuGet returned notes for {PackageId} v{Version} (length: {Length}): {Preview}...", packageId, version, releaseNotes.Length, releaseNotes[..Math.Min(150, releaseNotes.Length)]); - // Check if release notes are URL-dominated and try to resolve actual content var resolved = await _urlResolver.ResolveAsync(releaseNotes, packageId, version, githubRepoUrl); if (resolved.WasResolved) { @@ -59,7 +57,6 @@ public async Task GetReleaseNotesAsync( _logger.LogWarning("[NuGetFirstStrategy] NuGet returned empty for {PackageId} v{Version}", packageId, version); - // NuGet empty, try to discover GitHub URL if not known if (string.IsNullOrWhiteSpace(githubRepoUrl)) { _logger.LogDebug("[NuGetFirstStrategy] Discovering GitHub URL for {PackageId}", packageId); @@ -75,7 +72,6 @@ public async Task GetReleaseNotesAsync( } } - // Fallback to GitHub if (!string.IsNullOrWhiteSpace(githubRepoUrl)) { _logger.LogDebug("[NuGetFirstStrategy] Trying GitHub fallback for {PackageId} v{Version} from {GitHubUrl}", @@ -92,7 +88,6 @@ public async Task GetReleaseNotesAsync( _logger.LogWarning("[NuGetFirstStrategy] ALL SOURCES EMPTY for {PackageId} v{Version}", packageId, version); - // Persist failure to scrape logs for admin review var attemptedSources = !string.IsNullOrWhiteSpace(githubRepoUrl) ? "NuGet, GitHub" : "NuGet"; string? availableVersions = null; diff --git a/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs b/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs index d5387b5..a5ce791 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs @@ -36,7 +36,7 @@ public async Task> GetPackageVersionsAsync(string pack var metadataResource = await GetMetadataResourceAsync(); var metadata = await metadataResource.GetMetadataAsync( packageId, - includePrerelease: true, // Always fetch all versions, filter below + includePrerelease: true, includeUnlisted: false, _cache, NullLogger.Instance, @@ -44,10 +44,8 @@ public async Task> GetPackageVersionsAsync(string pack var versions = metadata.Select(m => m.Identity.Version); - // Get active exclude patterns var excludePatterns = await _versionFilterService.GetActiveExcludePatternsAsync(); - // Filter out prerelease versions if not requested if (!includePrerelease) { versions = versions @@ -57,7 +55,6 @@ public async Task> GetPackageVersionsAsync(string pack } else { - // Even when including prerelease, filter out excluded patterns (dev, ci, etc.) versions = versions .Where(v => !_versionFilterService.ShouldExcludeVersion(v.ToString(), excludePatterns)); } @@ -132,13 +129,11 @@ public async Task GetPackageMetadataAsync(string packageId) throw new ArgumentException($"Package '{packageId}' not found on NuGet."); } - // Get active exclude patterns var excludePatterns = await _versionFilterService.GetActiveExcludePatternsAsync(); _logger.LogDebug("[NugetService] Fetched {TotalCount} versions for {PackageId}", metadata.Count, packageId); - // Filter out excluded versions (dev, ci, etc.) var filteredMetadata = metadata .Where(m => !_versionFilterService.ShouldExcludeVersion(m.Identity.Version.ToString(), excludePatterns)) .ToList(); @@ -146,7 +141,6 @@ public async Task GetPackageMetadataAsync(string packageId) _logger.LogDebug("[NugetService] After filtering: {FilteredCount}/{TotalCount} versions remain for {PackageId}", filteredMetadata.Count, metadata.Count, packageId); - // Use filtered metadata for latest version var latest = filteredMetadata.OrderByDescending(m => m.Identity.Version).First(); return new PackageMetadata diff --git a/src/backend/PkgDiff.Infrastructure/Services/OsvClient.cs b/src/backend/PkgDiff.Infrastructure/Services/OsvClient.cs index 7b0ccd2..76fc82a 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/OsvClient.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/OsvClient.cs @@ -9,8 +9,7 @@ namespace PkgDiff.Infrastructure.Services; -// OSV.dev: which known vulnerabilities affect which version. One batch query names the advisories -// per version; the details of each advisory are then read once. https://google.github.io/osv.dev/api/ +// OSV.dev: one batch query names the advisories per version, then each advisory is read once public class OsvClient : IOsvClient { private const int BatchSize = 1000; @@ -191,8 +190,7 @@ private static string SummaryOf(Vulnerability vuln) return text.Length <= MaxSummaryLength ? text : text[..MaxSummaryLength] + "…"; } - // GitHub advisories carry a word (CRITICAL, HIGH, MODERATE, LOW); other databases may give only a - // CVSS vector, whose score is not computed here, so they stay "unknown" + // Only GitHub advisories carry a severity word; a bare CVSS vector is not scored, so stays unknown private static string SeverityOf(Vulnerability vuln) { var word = vuln.DatabaseSpecific is { ValueKind: JsonValueKind.Object } specific diff --git a/src/backend/PkgDiff.Infrastructure/Services/PackageHealthClient.cs b/src/backend/PkgDiff.Infrastructure/Services/PackageHealthClient.cs index 2a9e703..0050433 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/PackageHealthClient.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/PackageHealthClient.cs @@ -9,9 +9,8 @@ namespace PkgDiff.Infrastructure.Services; -// Project health: deps.dev for npm, PyPI and NuGet (licenses, provenance, OpenSSF Scorecard of the -// source repository); the Scorecard API for a GitHub repository deps.dev does not cover, which is -// how pub.dev packages get theirs. deps.dev data is CC BY 4.0, so its attribution travels with it. +// Project health from deps.dev, or the Scorecard API for repositories deps.dev does not cover. +// deps.dev data is CC BY 4.0, so its attribution travels with it. public partial class PackageHealthClient : IPackageHealthClient { private const string DepsDevAttribution = "Data from deps.dev, licensed under CC BY 4.0"; diff --git a/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs b/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs index 569c95c..4b1c979 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs @@ -34,7 +34,7 @@ public async Task GetReleaseNotesAsync(string packageId, string version) if (changelogNodes == null || !changelogNodes.Any()) return string.Empty; - var versionId = version.Replace(".", ""); // IDs in pub.dev seem to lack dots for some reason (e.g. #110 for 1.1.0) + var versionId = version.Replace(".", ""); // pub.dev changelog anchors drop the dots foreach (var node in changelogNodes) { @@ -46,13 +46,11 @@ public async Task GetReleaseNotesAsync(string packageId, string version) var rawHtml = contentNode.InnerHtml; - // Convert structural tags to markdown-like plain text before stripping rawHtml = Regex.Replace(rawHtml, "]*>", "\n", RegexOptions.IgnoreCase); rawHtml = Regex.Replace(rawHtml, "

", "\n\n", RegexOptions.IgnoreCase); rawHtml = Regex.Replace(rawHtml, "]*>", "\n- ", RegexOptions.IgnoreCase); rawHtml = Regex.Replace(rawHtml, "]*>", "\n# ", RegexOptions.IgnoreCase); - // Formatting tags rawHtml = Regex.Replace(rawHtml, "]*>(.*?)", "**$1**", RegexOptions.IgnoreCase | RegexOptions.Singleline); rawHtml = Regex.Replace(rawHtml, "]*>(.*?)", "**$1**", RegexOptions.IgnoreCase | RegexOptions.Singleline); rawHtml = Regex.Replace(rawHtml, "]*>(.*?)", "*$1*", RegexOptions.IgnoreCase | RegexOptions.Singleline); @@ -60,14 +58,12 @@ public async Task GetReleaseNotesAsync(string packageId, string version) rawHtml = Regex.Replace(rawHtml, "]*>(.*?)", "`$1`", RegexOptions.IgnoreCase | RegexOptions.Singleline); rawHtml = Regex.Replace(rawHtml, "]*>(.*?)", "\n```\n$1\n```\n", RegexOptions.IgnoreCase | RegexOptions.Singleline); - // Links rawHtml = Regex.Replace(rawHtml, "]+href=\"([^\"]+)\"[^>]*>(.*?)", "[$2]($1)", RegexOptions.IgnoreCase | RegexOptions.Singleline); var tempDoc = new HtmlDocument(); tempDoc.LoadHtml(rawHtml); var plainText = tempDoc.DocumentNode.InnerText; - // Clean up excessive newlines plainText = Regex.Replace(plainText, @"\n{3,}", "\n\n"); return plainText.Trim(); @@ -92,7 +88,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) var doc = new HtmlDocument(); doc.LoadHtml(html); - // Need to look for a link pointing to github.com specifically with text 'Repository (GitHub)' var links = doc.DocumentNode.SelectNodes("//a[@href]"); if (links != null) { @@ -114,7 +109,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } } - // Additional methods based on the pub.dev integration needs public async Task<(string? Publisher, string? Description, string? RepositoryUrl)> GetPackageExtraInfoAsync(string packageId) { string? publisher = null; @@ -128,7 +122,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) var doc = new HtmlDocument(); doc.LoadHtml(html); - // Extract Publisher from the var metaNode = doc.DocumentNode.SelectSingleNode("//meta[@name='pub-page-data']"); if (metaNode != null) { @@ -136,7 +129,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) if (!string.IsNullOrEmpty(base64Data)) { var jsonStr = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(base64Data)); - // Basic regex to avoid creating full JSON models if we just need one field. var match = Regex.Match(jsonStr, "\"publisherId\"\\s*:\\s*\"([^\"]+)\""); if (match.Success) { @@ -145,14 +137,12 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } } - // Extract Description var descNode = doc.DocumentNode.SelectSingleNode("//meta[@property='og:description']"); if (descNode != null) { description = descNode.GetAttributeValue("content", ""); } - // Extract Repository URL (GitHub) var links = doc.DocumentNode.SelectNodes("//a[@href]"); if (links != null) { diff --git a/src/backend/PkgDiff.Infrastructure/Services/PyPIService.cs b/src/backend/PkgDiff.Infrastructure/Services/PyPIService.cs index 7f376ac..e793e1c 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/PyPIService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/PyPIService.cs @@ -84,7 +84,6 @@ public async Task PackageExistsAsync(string packageId) var packageData = await GetPackageDataAsync(packageId); if (packageData?.Releases != null && packageData.Releases.TryGetValue(version, out var releaseFiles) && releaseFiles.Any()) { - // Return the upload time of the first file for this version return releaseFiles.First().UploadTime; } return null; @@ -142,13 +141,12 @@ private static bool IsGitHubUrl(string? url) private static readonly string[] RepositoryKeys = { "Source", "Source Code", "Source code", "Code", "Repository", "GitHub", "Github", "Homepage", "Home" }; - // Links that point at GitHub but not at the repository: sponsorship pages above all + // GitHub links that are not the repository, such as sponsorship pages private static readonly string[] NotRepositoryKeys = { "Funding", "Sponsor", "Sponsors", "Donate", "Donation" }; private static readonly string[] NotRepositoryOwners = { "sponsors", "orgs", "users", "apps", "marketplace", "settings" }; - // The repository among the links of a package: the ones labelled as the source first, then the home - // page, then any other GitHub link. "Funding" (github.com/sponsors/Kludex for uvicorn) never counts. + // Source-labelled links first, then the home page, then any other GitHub link; never funding private static string? ExtractGitHubUrl(PyPIPackageInfo? info) { if (info == null) @@ -183,8 +181,7 @@ private static bool IsGitHubUrl(string? url) private static string NormalizeName(string name) => System.Text.RegularExpressions.Regex.Replace(name, @"[-_.]+", "-").ToLowerInvariant(); - // https://github.com/{owner}/{repo} of a GitHub link (issues, a changelog file, releases), keeping a - // /tree/{ref}/{folder} part that names the package folder of a monorepo; null for any other link + // The repository URL of a GitHub link, keeping a /tree/{ref}/{folder} part (monorepo folder) private static string? RepositoryRoot(string? url) { if (!IsGitHubUrl(url)) @@ -233,7 +230,6 @@ private PackageMetadata MapToPackageMetadata(string packageId, PyPIPackageData p }; } - // PyPI API response models private class PyPIPackageData { [JsonPropertyName("info")] diff --git a/src/backend/PkgDiff.Infrastructure/Services/RateLimiterService.cs b/src/backend/PkgDiff.Infrastructure/Services/RateLimiterService.cs index a7ce087..2119d7a 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/RateLimiterService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/RateLimiterService.cs @@ -6,9 +6,6 @@ namespace PkgDiff.Infrastructure.Services; -/// -/// Rate limiter service using sliding window algorithm -/// public class RateLimiterService : IRateLimiterService { private const string GitHubApiKey = "github-api"; @@ -16,50 +13,44 @@ public class RateLimiterService : IRateLimiterService private readonly ILogger _logger; private readonly ConcurrentDictionary _buckets = new(); private readonly Dictionary _configs; - private readonly SemaphoreSlim _githubApiSemaphore; // Max concurrent GitHub API requests + private readonly SemaphoreSlim _githubApiSemaphore; public RateLimiterService(ILogger logger, IConfiguration configuration) { _logger = logger; - // Load rate limit configurations from appsettings var rateLimitSettings = configuration.GetSection("RateLimitSettings"); - // Configure max concurrent GitHub API requests (default: 50) var maxConcurrentGitHubRequests = rateLimitSettings.GetValue("GitHubApiMaxConcurrent", 50); _githubApiSemaphore = new SemaphoreSlim(maxConcurrentGitHubRequests, maxConcurrentGitHubRequests); _configs = new() { - // NuGet.org web scraping: 50 requests per minute (using 45 to be safe) + // nuget.org allows 50 requests per minute; the default stays below it ["nuget-scraping"] = new RateLimitConfig { MaxRequests = rateLimitSettings.GetValue("NuGetScrapingMaxRequests", 45), WindowSeconds = rateLimitSettings.GetValue("NuGetScrapingWindowSeconds", 60) }, - // NuGet.org V3 API: Not rate limited, but let's be respectful ["nuget-api"] = new RateLimitConfig { MaxRequests = rateLimitSettings.GetValue("NuGetApiMaxRequests", 1000), WindowSeconds = rateLimitSettings.GetValue("NuGetApiWindowSeconds", 60) }, - // NPM registry: Also generous ["npm-api"] = new RateLimitConfig { MaxRequests = rateLimitSettings.GetValue("NpmApiMaxRequests", 500), WindowSeconds = rateLimitSettings.GetValue("NpmApiWindowSeconds", 60) }, - // GitHub scraping: Be conservative (30 req/min) ["github-scraping"] = new RateLimitConfig { MaxRequests = rateLimitSettings.GetValue("GitHubScrapingMaxRequests", 30), WindowSeconds = rateLimitSettings.GetValue("GitHubScrapingWindowSeconds", 60) }, - // GitHub API: Higher limit with authentication, managed by concurrent semaphore [GitHubApiKey] = new RateLimitConfig { MaxRequests = rateLimitSettings.GetValue("GitHubApiMaxRequests", 5000), @@ -73,7 +64,6 @@ public async Task ExecuteAsync(string key, Func> action) var bucket = _buckets.GetOrAdd(key, k => new RateLimitBucket()); var config = _configs.GetValueOrDefault(key, new RateLimitConfig { MaxRequests = 100, WindowSeconds = 60 }); - // For github-api, use concurrent request limiting with semaphore if (key == GitHubApiKey) { await _githubApiSemaphore.WaitAsync(); @@ -103,7 +93,7 @@ private async Task ExecuteWithRateLimitAsync(string key, RateLimitBucket b "[RateLimiter] Rate limit reached for '{Key}'. Waiting {WaitSeconds:F2}s. ({Max} requests in {Window}s window)", key, waitTime.TotalSeconds, config.MaxRequests, config.WindowSeconds); - // The gate stays held during the wait, so callers of this key leave one by one instead of all at once + // The gate stays held during the wait so callers of this key leave one by one await Task.Delay(waitTime, WorkCancellation.Token); } @@ -114,7 +104,6 @@ private async Task ExecuteWithRateLimitAsync(string key, RateLimitBucket b bucket.Gate.Release(); } - // Execute the action outside the gate return await action(); } diff --git a/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesRenderer.cs b/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesRenderer.cs index 3c59ac3..639ffee 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesRenderer.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesRenderer.cs @@ -20,8 +20,7 @@ namespace PkgDiff.Infrastructure.Services; -// Markdown is parsed once, here, and only the sanitized HTML leaves the server: no second parser can -// disagree with this one about what is code and what is markup. +// Only sanitized HTML leaves the server, so no second parser can disagree about code vs markup public sealed class ReleaseNotesRenderer : IReleaseNotesRenderer, IDisposable { private const string LinkRel = "noopener noreferrer nofollow"; @@ -38,8 +37,7 @@ public sealed class ReleaseNotesRenderer : IReleaseNotesRenderer, IDisposable private static readonly string[] AllowedSchemes = ["http", "https", "mailto"]; - // An image loads from where the package author pointed it, telling that host who reads the notes. - // These hosts are trusted with that; an image anywhere else is shown as a link the reader may open. + // Images elsewhere become links: loading one tells that host who reads the notes private static readonly string[] TrustedImageHosts = ["github.com", "githubusercontent.com", "img.shields.io", "badgen.net"]; internal static readonly HashSet AllowedTagSet = new(AllowedTags, StringComparer.OrdinalIgnoreCase); @@ -147,8 +145,7 @@ public string RenderHtml(string markdown) if (string.IsNullOrWhiteSpace(markdown)) return string.Empty; - // Rendering costs milliseconds per note and a package page renders every cached note, so identical - // input is served from a size-bounded cache keyed by its hash + // A package page renders every cached note, so output is cached by input hash var key = Convert.ToHexString(SHA256.HashData(MemoryMarshal.AsBytes(markdown.AsSpan()))); if (_cache.TryGetValue(key, out string? cached) && cached is not null) return cached; @@ -157,8 +154,7 @@ public string RenderHtml(string markdown) var limit = MarkdownComplexityGuard.Check(markdown, out var measured); if (limit != MarkdownComplexityGuard.Limit.None) { - // A stack overflow in the parsers ends the process and no catch block can stop it, so input - // that could cause one is never parsed + // A stack overflow cannot be caught, so input that could cause one is never parsed _logger.LogWarning( "Release notes exceed the {Limit} limit ({Measured}) at {Length} characters, showing them as plain text", limit, measured, markdown.Length); @@ -256,7 +252,7 @@ private static void RewriteInlineHtml(MarkdownDocument document) continue; } - // Not allowlisted: the tag stays visible as text (e.g. the of List) + // Not allowlisted: the tag stays visible as text html.ReplaceBy(new LiteralInline(tag)); } } diff --git a/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesUrlResolver.cs b/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesUrlResolver.cs index 3d9121c..3b8500b 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesUrlResolver.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ReleaseNotesUrlResolver.cs @@ -15,35 +15,29 @@ public class ReleaseNotesUrlResolver : IReleaseNotesUrlResolver private readonly AsyncRetryPolicy _retryPolicy; private readonly ILogger _logger; - // URL extraction regex private static readonly Regex UrlRegex = new( @"https?://[^\s""'<>\)]+", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - // GitHub releases page: github.com/{owner}/{repo}/releases private static readonly Regex GitHubReleasesRegex = new( @"^https?://github\.com/(?[^/]+)/(?[^/]+)/releases/?$", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - // GitHub blob file: github.com/{owner}/{repo}/blob/{branch}/{path} private static readonly Regex GitHubBlobRegex = new( @"^https?://github\.com/(?[^/]+)/(?[^/]+)/blob/(?[^/]+)/(?.+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - // GitHub repo root: github.com/{owner}/{repo} private static readonly Regex GitHubRepoRegex = new( @"^https?://github\.com/(?[^/]+)/(?[^/]+?)(?:\.git)?/?$", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - // Changelog-like filenames private static readonly string[] ChangelogFilenames = ["changelog", "changes", "release-notes", "releasenotes", "history"]; - // Filler phrases commonly found around URL-only release notes private static readonly string[] FillerPhrases = [ "see", "please see", "please read", "read the", @@ -65,7 +59,7 @@ public ReleaseNotesUrlResolver( _httpClient = httpClient; _logger = logger; - // Only a transient failure is retried: a 404 or a blocked (SSRF) target stays what it is. + // Only transient failures are retried; a 404 or an SSRF-blocked target is final _retryPolicy = Policy .Handle(TransientHttp.IsTransient) .WaitAndRetryAsync(2, retryAttempt => @@ -92,7 +86,7 @@ public async Task ResolveAsync( "[UrlResolver] Detected URL-dominated release notes for {PackageId} v{Version}, attempting resolution...", packageId, version); - // The first few links only: each one may wait for a slow host, and a note can list hundreds + // Capped: each link may wait on a slow host and a note can list hundreds var classifiedUrls = ClassifyUrls(urls).Take(MaxUrlsFollowed); foreach (var (url, urlType) in classifiedUrls) @@ -110,7 +104,7 @@ public async Task ResolveAsync( } catch (GitHubRateLimitedException) { - // The link may still resolve: the version is scraped again once the limit resets + // Rethrown so the version is scraped again once the rate limit resets throw; } catch (Exception ex) @@ -124,8 +118,6 @@ public async Task ResolveAsync( return new UrlResolverResult(rawReleaseNotes, false); } - // ==================== URL EXTRACTION & CLASSIFICATION ==================== - private const int MaxUrlsFollowed = 3; private static List ExtractUrls(string text) @@ -136,10 +128,6 @@ private static List ExtractUrls(string text) .ToList(); } - /// - /// Determines whether release notes text is "URL-dominated" - minimal substantive - /// content beyond URLs and filler phrases. - /// private static bool IsUrlDominated(string text, List urls) { var strippedText = text; @@ -150,15 +138,11 @@ private static bool IsUrlDominated(string text, List urls) foreach (var filler in FillerPhrases) remaining = Regex.Replace(remaining, Regex.Escape(filler), "", RegexOptions.IgnoreCase); - // Strip punctuation and whitespace remaining = Regex.Replace(remaining, @"[^\w]", ""); return remaining.Length < 80; } - /// - /// Classifies URLs by type and sorts by resolution priority. - /// private static List<(string Url, UrlType Type)> ClassifyUrls(List urls) { var classified = new List<(string Url, UrlType Type)>(); @@ -189,8 +173,6 @@ private static bool IsChangelogPath(string url) return ChangelogFilenames.Any(name => path.Contains(name)); } - // ==================== URL RESOLUTION HANDLERS ==================== - private async Task ResolveUrlAsync( string url, UrlType urlType, string version) { @@ -205,9 +187,6 @@ private static bool IsChangelogPath(string url) }; } - /// - /// GitHub releases page → delegate to existing IGitHubScraperService - /// private async Task ResolveGitHubReleasesAsync(string url, string version) { var repoUrl = url.Replace("/releases", "").TrimEnd('/'); @@ -219,9 +198,6 @@ private static bool IsChangelogPath(string url) return null; } - /// - /// Bare GitHub repo URL → try releases page - /// private async Task ResolveGitHubRepoAsync(string url, string version) { var repoUrl = url.TrimEnd('/'); @@ -233,9 +209,6 @@ private static bool IsChangelogPath(string url) return null; } - /// - /// GitHub blob CHANGELOG → convert to raw URL, fetch, extract version section - /// private async Task ResolveGitHubChangelogAsync(string url, string version) { return await _retryPolicy.ExecuteAsync(async () => @@ -261,9 +234,6 @@ private static bool IsChangelogPath(string url) }); } - /// - /// Non-changelog GitHub blob file (e.g., upgrade guide) - /// private async Task ResolveGitHubBlobAsync(string url, string version) { return await _retryPolicy.ExecuteAsync(async () => @@ -274,7 +244,6 @@ private static bool IsChangelogPath(string url) var content = await _httpClient.GetStringAsync(rawUrl); if (string.IsNullOrWhiteSpace(content)) return null; - // Try version section first, fallback to full content var versionSection = ExtractVersionSection(content, version); var result = !string.IsNullOrWhiteSpace(versionSection) ? versionSection : content; @@ -285,9 +254,6 @@ private static bool IsChangelogPath(string url) }); } - /// - /// External (non-GitHub) URL → fetch HTML, extract main content - /// private async Task ResolveExternalPageAsync(string url) { return await _retryPolicy.ExecuteAsync(async () => @@ -306,7 +272,7 @@ private static bool IsChangelogPath(string url) ".post-content", ".entry-content", ".document-content", - ".rst-content" // ReadTheDocs + ".rst-content" }; foreach (var selector in contentSelectors) @@ -329,11 +295,6 @@ private static bool IsChangelogPath(string url) }); } - // ==================== HELPERS ==================== - - /// - /// Converts github.com blob URL to raw.githubusercontent.com URL - /// private static string? ConvertToRawGitHubUrl(string blobUrl) { var match = GitHubBlobRegex.Match(blobUrl); @@ -347,11 +308,6 @@ private static bool IsChangelogPath(string url) return $"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}"; } - /// - /// Extracts the section of a CHANGELOG markdown file for a specific version. - /// Looks for headings like: ## [1.2.3], ## v1.2.3, ## 1.2.3 - 2024-01-01 - /// Returns content between that heading and the next heading of same/higher level. - /// internal static string? ExtractVersionSection(string changelogContent, string version) { var cleanVersion = version.TrimStart('v', 'V'); diff --git a/src/backend/PkgDiff.Infrastructure/Services/ScraperService.cs b/src/backend/PkgDiff.Infrastructure/Services/ScraperService.cs index 2152bae..f7fab33 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/ScraperService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/ScraperService.cs @@ -17,7 +17,7 @@ public class ScraperService : IScraperService private readonly IRateLimiterService _rateLimiter; private readonly ILogger _logger; - // Request deduplication: prevent multiple concurrent requests for the same resource + // In-flight requests, shared so concurrent callers trigger a single fetch private readonly ConcurrentDictionary> _releaseNotesCache = new(); private readonly ConcurrentDictionary> _githubUrlCache = new(); @@ -35,7 +35,6 @@ public ScraperService( _htmlDocumentLoader = htmlDocumentLoader; _httpClientFactory = httpClientFactory; - // Polly retry policy: 3 attempts with exponential backoff, for transient failures only _retryPolicy = Policy .Handle(TransientHttp.IsTransient) .WaitAndRetryAsync(3, retryAttempt => @@ -46,14 +45,12 @@ public async Task GetReleaseNotesAsync(string packageId, string version) { var cacheKey = $"{packageId}:{version}"; - // Request deduplication: if there's already a pending request for this package+version, reuse it return await _releaseNotesCache.GetOrAdd(cacheKey, async key => { try { return await _retryPolicy.ExecuteAsync(async () => { - // Use rate limiter to respect NuGet.org's 50 req/min limit return await _rateLimiter.ExecuteAsync("nuget-scraping", async () => { var url = $"https://www.nuget.org/packages/{PackageUrlEncoder.EscapeSegment(packageId)}/{PackageUrlEncoder.EscapeSegment(version)}"; @@ -65,8 +62,7 @@ public async Task GetReleaseNotesAsync(string packageId, string version) return string.Empty; } - // Try multiple selectors as fallback - // NOTE: Use #releasenotes-tab (content div), NOT #releasenotes-body-tab (tab button) + // #releasenotes-tab is the content div; #releasenotes-body-tab is the tab button var selectors = new[] { "#releasenotes-tab p", @@ -92,7 +88,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } finally { - // Remove from cache after completion to allow fresh requests later _releaseNotesCache.TryRemove(cacheKey, out _); } }); @@ -100,17 +95,14 @@ public async Task GetReleaseNotesAsync(string packageId, string version) public async Task GetGitHubRepositoryUrlAsync(string packageId) { - // Request deduplication: if there's already a pending request for this package, reuse it return await _githubUrlCache.GetOrAdd(packageId, async key => { try { return await _retryPolicy.ExecuteAsync(async () => { - // Use rate limiter to respect NuGet.org's 50 req/min limit return await _rateLimiter.ExecuteAsync("nuget-scraping", async () => { - // Try HTML scraping first (faster and simpler) var githubUrl = await ScrapeGitHubUrlFromHtmlAsync(packageId); if (!string.IsNullOrWhiteSpace(githubUrl)) { @@ -124,7 +116,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } finally { - // Remove from cache after completion to allow fresh requests later _githubUrlCache.TryRemove(packageId, out _); } }); @@ -134,7 +125,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) { try { - // Use NuGet V3 API to get package metadata var url = $"https://api.nuget.org/v3/registration5-gz-semver2/{PackageUrlEncoder.EscapeSegment(packageId.ToLowerInvariant())}/index.json"; var httpClient = _httpClientFactory.CreateClient(); @@ -146,19 +136,16 @@ public async Task GetReleaseNotesAsync(string packageId, string version) var json = await response.Content.ReadAsStringAsync(); using var doc = System.Text.Json.JsonDocument.Parse(json); - // Get the last page (contains latest versions) var items = doc.RootElement.GetProperty("items"); if (items.GetArrayLength() == 0) return null; var lastPage = items[items.GetArrayLength() - 1]; - // Get items from the page var pageItems = lastPage.GetProperty("items"); if (pageItems.GetArrayLength() == 0) return null; - // Get the last item (latest version) var latestItem = pageItems[pageItems.GetArrayLength() - 1]; var catalogEntry = latestItem.GetProperty("catalogEntry"); var version = catalogEntry.GetProperty("version").GetString(); @@ -172,9 +159,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } } - /// - /// Extracts GitHub repository URL from .nuspec XML content. - /// private string? ExtractGitHubUrlFromNuspec(string nuspecContent) { try @@ -184,7 +168,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) if (ns != null) { - // First, try tag (most specific) var repositoryElement = xmlDoc.Descendants(ns + "repository").FirstOrDefault(); var url = repositoryElement?.Attribute("url")?.Value; @@ -194,7 +177,6 @@ public async Task GetReleaseNotesAsync(string packageId, string version) return url; } - // Fallback: Try tag (some packages put GitHub URL here) var projectUrlElement = xmlDoc.Descendants(ns + "projectUrl").FirstOrDefault(); var projectUrl = projectUrlElement?.Value; @@ -221,7 +203,6 @@ private string CleanGitHubUrl(string url) url = url.TrimEnd('/'); - // Remove common GitHub paths to get base repository URL var segments = new[] { "/issues", "/wiki", "/pulls", "/actions", "/tree", "/blob", "/commit" }; foreach (var segment in segments) { @@ -241,20 +222,13 @@ private string CleanReleaseNotes(string text) if (string.IsNullOrWhiteSpace(text)) return string.Empty; - // Remove excessive whitespace text = System.Text.RegularExpressions.Regex.Replace(text, @"\s+", " "); - // Trim text = text.Trim(); return text; } - // ==================== HELPER METHODS ==================== - - /// - /// Scrapes GitHub repository URL from NuGet package HTML page. - /// private async Task ScrapeGitHubUrlFromHtmlAsync(string packageId) { try @@ -266,7 +240,6 @@ private string CleanReleaseNotes(string text) return null; } - // Try the specific selector from the actual HTML var githubLink = document.QuerySelector("a[data-track='outbound-repository-url'][href*='github.com']"); if (githubLink != null) @@ -279,7 +252,6 @@ private string CleanReleaseNotes(string text) } } - // Fallback 1: Try any link with "Source repository" text var links = document.QuerySelectorAll("a[href*='github.com']"); foreach (var link in links) { @@ -295,7 +267,6 @@ private string CleanReleaseNotes(string text) } } - // Fallback 2: Try "Project website" link (some packages put GitHub URL here) foreach (var link in links) { var text = link.TextContent?.Trim(); @@ -310,8 +281,6 @@ private string CleanReleaseNotes(string text) } } - // Developer note: if HTML needs to be inspected during dev, handle it securely here. - return null; } catch (Exception ex) diff --git a/src/backend/PkgDiff.Infrastructure/Services/SocialMediaService.cs b/src/backend/PkgDiff.Infrastructure/Services/SocialMediaService.cs index 466090f..d95910d 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/SocialMediaService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/SocialMediaService.cs @@ -84,7 +84,6 @@ public async Task PostToTwitterAsync(string content) client = _httpClientFactory.CreateClient("Twitter"); contentPayload = new StringContent(JsonSerializer.Serialize(new { text = content }), Encoding.UTF8, "application/json"); - // Generate OAuth 1.0a Authorization header string authHeader = GenerateTwitterOAuthHeader(requestUrl, "POST", apiKey, apiKeySecret, accessToken, accessTokenSecret); client.DefaultRequestHeaders.Add("Authorization", authHeader); } @@ -128,7 +127,7 @@ public async Task PostToLinkedInAsync(string content) try { var accessToken = await _settingsReader.GetAsync(SettingKeys.LinkedInAccessToken); - var authorURN = await _settingsReader.GetAsync(SettingKeys.LinkedInAuthorUrn); // e.g. urn:li:organization:1234567 or urn:li:person:abcdef + var authorURN = await _settingsReader.GetAsync(SettingKeys.LinkedInAuthorUrn); if (string.IsNullOrEmpty(accessToken) || string.IsNullOrEmpty(authorURN)) { @@ -138,7 +137,7 @@ public async Task PostToLinkedInAsync(string content) client = _httpClientFactory.CreateClient("LinkedIn"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); client.DefaultRequestHeaders.Add("X-Restli-Protocol-Version", "2.0.0"); - client.DefaultRequestHeaders.Add("LinkedIn-Version", "202602"); // Latest active version (Feb 2026) + client.DefaultRequestHeaders.Add("LinkedIn-Version", "202602"); var payload = new { @@ -170,19 +169,18 @@ public async Task PostToLinkedInAsync(string content) using var response = sent.Response!; if (response.IsSuccessStatusCode) { - // LinkedIn returns 201 Created and Post ID in the x-restli-id header if (response.Headers.TryGetValues("x-restli-id", out var values)) { var postId = values.FirstOrDefault(); return SocialPostResult.Succeeded($"https://www.linkedin.com/feed/update/{postId}"); } - return SocialPostResult.Succeeded("https://www.linkedin.com/company/pkgdiff"); // Fallback URL + return SocialPostResult.Succeeded("https://www.linkedin.com/company/pkgdiff"); } return await FailureFromResponseAsync(response, "LinkedIn"); } - // Once the request has started, a timeout or broken connection says nothing about whether the platform published it + // Once sent, a timeout or broken connection does not tell whether the post was published private async Task<(HttpResponseMessage? Response, SocialPostResult? Result)> SendAsync( HttpClient client, string requestUrl, @@ -263,8 +261,7 @@ await _cache.SetStringAsync( JsonSerializer.Serialize(new OAuthStateEntry(userId, redirectUri)), new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = OAuthStateLifetime }); - // Only request standard scopes that are available to all LinkedIn apps - // w_member_social: post on behalf of member, openid/profile/email: basic profile + // Only standard scopes, available to every LinkedIn app var scopes = "openid profile email w_member_social"; var authUrl = $"https://www.linkedin.com/oauth/v2/authorization" + $"?response_type=code" + @@ -312,7 +309,6 @@ await _cache.SetStringAsync( if (string.IsNullOrEmpty(clientId) || string.IsNullOrEmpty(clientSecret)) return (false, "LinkedIn ClientId or ClientSecret is not configured in System Settings."); - // Step 1: Exchange authorization code for access token var tokenClient = _httpClientFactory.CreateClient(); var tokenRequest = new FormUrlEncodedContent(new Dictionary { @@ -341,13 +337,11 @@ await _cache.SetStringAsync( if (string.IsNullOrEmpty(accessToken)) return (false, "Access token was empty in LinkedIn's response."); - // Step 2: Find the managed company page (pkgdiff) var orgClient = _httpClientFactory.CreateClient(); orgClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); orgClient.DefaultRequestHeaders.Add("X-Restli-Protocol-Version", "2.0.0"); - orgClient.DefaultRequestHeaders.Add("LinkedIn-Version", "202602"); // Latest active version (Feb 2026) + orgClient.DefaultRequestHeaders.Add("LinkedIn-Version", "202602"); - // Get organizations where the user is an admin var orgsResponse = await orgClient.GetAsync("https://api.linkedin.com/v2/organizationAcls?q=roleAssignee&role=ADMINISTRATOR&state=APPROVED&count=50"); var orgsContent = await orgsResponse.Content.ReadAsStringAsync(); @@ -357,7 +351,6 @@ await _cache.SetStringAsync( using var orgsDoc = JsonDocument.Parse(orgsContent); var elements = orgsDoc.RootElement.GetProperty("elements"); - // Try to find pkgdiff org, otherwise fall back to first admin org foreach (var element in elements.EnumerateArray()) { if (element.TryGetProperty("organization", out var orgProp)) @@ -365,7 +358,6 @@ await _cache.SetStringAsync( var orgUrn = orgProp.GetString(); if (!string.IsNullOrEmpty(orgUrn)) { - // Fetch org details to find pkgdiff var orgDetailResp = await orgClient.GetAsync($"https://api.linkedin.com/v2/organizations/{orgUrn.Split(':').Last()}"); if (orgDetailResp.IsSuccessStatusCode) { @@ -379,14 +371,12 @@ await _cache.SetStringAsync( } } - // Fallback: Use first org if pkgdiff not found yet authorUrn ??= orgUrn; } } } } - // Step 3: Save AccessToken and AuthorURN to SystemSettings await SaveOrUpdateSettingAsync(SettingKeys.LinkedInAccessToken, accessToken, "LinkedIn OAuth Access Token (auto-generated)"); if (!string.IsNullOrEmpty(authorUrn)) diff --git a/src/backend/PkgDiff.Infrastructure/Services/SocketBehaviorClient.cs b/src/backend/PkgDiff.Infrastructure/Services/SocketBehaviorClient.cs index da0b8f0..f30b678 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/SocketBehaviorClient.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/SocketBehaviorClient.cs @@ -10,11 +10,8 @@ namespace PkgDiff.Infrastructure.Services; -// Socket.dev behaviour analysis (install scripts, network or shell access, obfuscation, malware) for -// npm, PyPI and NuGet. Off unless Security:Socket:Enabled is true and a token and organization are -// set: Socket's terms for showing its data publicly are not settled yet (docs/ops/socket-permission.md). -// The free tier allows about five batch calls an hour, so one call covers the newest versions of a -// package and this process makes at most four calls in any hour. +// Socket.dev behaviour analysis for npm, PyPI and NuGet; off unless enabled with a token and org. +// The free tier allows about five batch calls an hour, so calls are batched and capped. public partial class SocketBehaviorClient : IBehaviorClient { private const int MaxVersionsPerCall = 20; diff --git a/src/backend/PkgDiff.Infrastructure/Services/TransientHttp.cs b/src/backend/PkgDiff.Infrastructure/Services/TransientHttp.cs index 43db73a..557f956 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/TransientHttp.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/TransientHttp.cs @@ -5,9 +5,8 @@ namespace PkgDiff.Infrastructure.Services; public static class TransientHttp { - // Worth another attempt: the connection failed, the server answered 408, 429 or 5xx, or the - // client's own timeout fired. Not worth one: an answer the server gave on purpose (404, 403), - // a target the SSRF guard blocked, a request the caller cancelled, or a bug in our own code. + // Retry connection failures, 408/429/5xx and client timeouts; not deliberate answers, + // SSRF-blocked targets, caller cancellation or bugs in our own code public static bool IsTransient(Exception exception) { if (exception is SsrfBlockedException || exception.InnerException is SsrfBlockedException) diff --git a/src/backend/PkgDiff.Infrastructure/Services/WorkCancellationHandler.cs b/src/backend/PkgDiff.Infrastructure/Services/WorkCancellationHandler.cs index d7d684c..ed21aff 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/WorkCancellationHandler.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/WorkCancellationHandler.cs @@ -2,8 +2,7 @@ namespace PkgDiff.Infrastructure.Services; -// Added to every HTTP client: an outbound request ends when the work it belongs to is cancelled, -// whether or not the caller passed a token of its own. +// Ends every outbound request when its work is cancelled, even without a caller token public sealed class WorkCancellationHandler : DelegatingHandler { protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/src/backend/PkgDiff.Shared/Constants/Permissions.cs b/src/backend/PkgDiff.Shared/Constants/Permissions.cs index e6a7887..1a1cdcc 100644 --- a/src/backend/PkgDiff.Shared/Constants/Permissions.cs +++ b/src/backend/PkgDiff.Shared/Constants/Permissions.cs @@ -87,7 +87,6 @@ public static class SocialMedia public const string Manage = "Permissions.SocialMedia.Manage"; } - // Helper to get all permissions public static List GetAllPermissions() { var permissions = new List(); diff --git a/src/backend/PkgDiff.Shared/Constants/RegistryNames.cs b/src/backend/PkgDiff.Shared/Constants/RegistryNames.cs index 9f0c8f6..e79ae27 100644 --- a/src/backend/PkgDiff.Shared/Constants/RegistryNames.cs +++ b/src/backend/PkgDiff.Shared/Constants/RegistryNames.cs @@ -1,6 +1,5 @@ namespace PkgDiff.Shared.Constants; -// The registries the application serves, by the name used in URLs and in the Registries table public static class RegistryNames { public const string NuGet = "nuget"; @@ -10,14 +9,12 @@ public static class RegistryNames public static readonly IReadOnlyList All = new[] { NuGet, Npm, PyPI, PubDev }; - // Other spellings a registry is also known by private static readonly IReadOnlyDictionary Aliases = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["pub"] = PubDev }; - // The registry a spelling stands for; null when it is none of them public static string? Canonical(string? registry) { if (string.IsNullOrEmpty(registry)) @@ -29,7 +26,7 @@ public static class RegistryNames return All.FirstOrDefault(name => name.Equals(registry, StringComparison.OrdinalIgnoreCase)); } - // Every spelling of the same registry: its row may be stored under any of them + // A registry's row may be stored under any of its spellings public static IReadOnlyList SpellingsOf(string registry) { var canonical = Canonical(registry); diff --git a/src/backend/PkgDiff.Shared/DTOs/AdminModels.cs b/src/backend/PkgDiff.Shared/DTOs/AdminModels.cs index bc2f2a2..a2ef107 100644 --- a/src/backend/PkgDiff.Shared/DTOs/AdminModels.cs +++ b/src/backend/PkgDiff.Shared/DTOs/AdminModels.cs @@ -64,7 +64,6 @@ public class PaginatedResult public int TotalPages { get; set; } } -// Admin - Packages public class AdminPackageInfo { public int Id { get; set; } @@ -76,7 +75,6 @@ public class AdminPackageInfo public DateTime ExpiresAt { get; set; } } -// Admin - Registries public class AdminRegistryInfo { public int Id { get; set; } @@ -106,7 +104,6 @@ public class RegistryUpdateRequest public int DisplayOrder { get; set; } } -// Admin - Jobs public class JobsOverview { public long Enqueued { get; set; } @@ -152,7 +149,6 @@ public class QueuedJobInfo public string? PackageId { get; set; } } -// Admin - Settings public class SystemSettingDto { public int Id { get; set; } @@ -171,7 +167,6 @@ public class UpdateSettingRequest public bool Clear { get; set; } } -// Admin - Users public class AdminUserInfo { public string Id { get; set; } = string.Empty; @@ -206,7 +201,6 @@ public class ResetUserPasswordRequest public string NewPassword { get; set; } = string.Empty; } -// Admin - Roles public class AdminRoleInfo { public string Id { get; set; } = string.Empty; @@ -219,7 +213,6 @@ public class CreateRoleRequest public string Name { get; set; } = string.Empty; } -// Admin - Security & IP Blocking public class RequestLogInfo { public int Id { get; set; } @@ -261,7 +254,6 @@ public class IpStatistics public bool IsBlocked { get; set; } } -// Admin - Scrape Logs public class ScrapeLogInfo { public int Id { get; set; } @@ -291,7 +283,6 @@ public class ScrapeLogPackageSummary public DateTime LastOccurrence { get; set; } } -// Permissions public class PermissionDto { public string Name { get; set; } = string.Empty; @@ -313,7 +304,6 @@ public class UpdateRolePermissionsRequest public List Permissions { get; set; } = new(); } -// Sitemap public class SitemapEntry { public string PackageId { get; set; } = string.Empty; diff --git a/src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs b/src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs index f635e4e..fc14471 100644 --- a/src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs +++ b/src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs @@ -67,7 +67,6 @@ public class PackageMetadata public string? IconUrl { get; set; } public string? ProjectUrl { get; set; } public string? GitHubRepositoryUrl { get; set; } - // The folder of the package inside its repository, for a package of a monorepo public string? SourceDirectory { get; set; } public List Versions { get; set; } = new(); public PackageTrust? Trust { get; set; } @@ -81,7 +80,6 @@ public class VersionSummary public string ReleaseNotes { get; set; } = string.Empty; public string ReleaseNotesHtml { get; set; } = string.Empty; public string? ReleaseNotesSource { get; set; } - // Set when pkgdiff compiled the notes itself (CompiledReleaseNotes) public CompiledNotesSource? CompiledFrom { get; set; } public List Entries { get; set; } = new(); public bool IsEmpty { get; set; } = true; @@ -94,7 +92,6 @@ public class VersionInfo public string ReleaseNotes { get; set; } = string.Empty; public string ReleaseNotesHtml { get; set; } = string.Empty; public string? ReleaseNotesSource { get; set; } - // Set when pkgdiff compiled the notes itself (CompiledReleaseNotes) public CompiledNotesSource? CompiledFrom { get; set; } public List Entries { get; set; } = new(); public List Categories { get; set; } = new(); @@ -102,7 +99,6 @@ public class VersionInfo public bool IsPrerelease { get; set; } } -// Where notes pkgdiff compiled itself come from, e.g. "18.3.0 milestone" and its GitHub URL public class CompiledNotesSource { public string Label { get; set; } = string.Empty; diff --git a/src/backend/PkgDiff.Shared/DTOs/AuditModels.cs b/src/backend/PkgDiff.Shared/DTOs/AuditModels.cs index aa8556c..e49d6bd 100644 --- a/src/backend/PkgDiff.Shared/DTOs/AuditModels.cs +++ b/src/backend/PkgDiff.Shared/DTOs/AuditModels.cs @@ -15,9 +15,8 @@ public static class AuditItemStatus { public const string Vulnerable = "vulnerable"; public const string Clean = "clean"; - // The registry does not know the package (a private or misspelled one) public const string NotFound = "not-found"; - // The version could not be checked (the registry could not be reached) + // The registry could not be reached public const string Unchecked = "unchecked"; } @@ -26,7 +25,6 @@ public class ManifestParseRequest [Required] public string Registry { get; set; } = string.Empty; - // The pasted file; 1 MB is far more than any real manifest or lock file of 200 packages needs [Required] [StringLength(AuditLimits.MaxContentLength)] public string Content { get; set; } = string.Empty; @@ -47,13 +45,11 @@ public class AuditDependency { public string Name { get; set; } = string.Empty; - // The version that is checked; null means "the latest version the registry has" + // null means the latest version the registry has public string? Version { get; set; } - // What the file says, e.g. "^4.17.1" or ">=2.0,<3" public string? Requirement { get; set; } - // True when the file pins exactly this version (lock files, "==1.2.3", "[1.2.3]") public bool IsExact { get; set; } // "dev", "optional" or null for a normal dependency @@ -64,7 +60,6 @@ public class ManifestParseResult { public string Registry { get; set; } = string.Empty; - // The detected file type, e.g. "package.json" or "requirements.txt" public string Format { get; set; } = string.Empty; public List Dependencies { get; set; } = new(); @@ -163,7 +158,6 @@ public class AuditItem public string? Scope { get; set; } public bool IsExact { get; set; } - // The version that was checked public string? Version { get; set; } // True when the file named no usable version and the latest one was checked instead @@ -173,11 +167,9 @@ public class AuditItem public string? LatestVersion { get; set; } public List Advisories { get; set; } = new(); - // The lowest version that fixes every advisory of the checked version; null when one of them - // has no fixed version yet (or there is nothing to fix) + // null when an advisory has no fixed version yet, or there is nothing to fix public string? SafeVersion { get; set; } - // True when at least one advisory has no fixed version published public bool HasUnfixedAdvisory { get; set; } public bool IsMalicious { get; set; } diff --git a/src/backend/PkgDiff.Shared/DTOs/SecurityModels.cs b/src/backend/PkgDiff.Shared/DTOs/SecurityModels.cs index d6990ed..42ab9b1 100644 --- a/src/backend/PkgDiff.Shared/DTOs/SecurityModels.cs +++ b/src/backend/PkgDiff.Shared/DTOs/SecurityModels.cs @@ -2,7 +2,6 @@ namespace PkgDiff.Shared.DTOs; public static class TrustStatus { - // No security data yet: a background job is collecting it public const string Pending = "pending"; public const string Ready = "ready"; // The providers are switched off, or the last refresh failed with nothing stored before it @@ -18,23 +17,20 @@ public static class AdvisorySeverity public const string Unknown = "unknown"; } -// The security picture of one package: known vulnerabilities per version (OSV) and the health of -// its project (deps.dev, OpenSSF Scorecard). Served from a cache filled by a background job. +// Served from a cache filled by a background job public class PackageTrust { public string Status { get; set; } = TrustStatus.Pending; public DateTime? FetchedAt { get; set; } - // Every advisory that affects at least one checked version, once public List Advisories { get; set; } = new(); - // Version string (as the registry spells it) -> ids of the advisories that affect it. A checked - // version without advisories has an empty list; a version that was not checked is missing. + // Checked version -> advisory ids (empty when clean); unchecked versions are absent public Dictionary> VersionAdvisories { get; set; } = new(); public PackageHealth? Health { get; set; } - // Behaviour analysis per version (Socket.dev); null while that provider is switched off + // null while the behaviour provider (Socket.dev) is switched off public Dictionary? Behavior { get; set; } } @@ -46,14 +42,13 @@ public class SecurityAdvisory public string Severity { get; set; } = AdvisorySeverity.Unknown; public List FixedIn { get; set; } = new(); public string Url { get; set; } = string.Empty; - // An OpenSSF malicious-packages record (id MAL-...): the version is malware, not merely vulnerable + // OpenSSF malicious-packages record (MAL-...): malware, not merely vulnerable public bool IsMalicious { get; set; } public DateTime? Published { get; set; } } public class PackageHealth { - // "deps.dev" or "OpenSSF Scorecard": where the figures come from, shown with its attribution public string Source { get; set; } = string.Empty; public string? SourceUrl { get; set; } public string? Attribution { get; set; } @@ -62,7 +57,7 @@ public class PackageHealth public DateTime? ScorecardDate { get; set; } public List Checks { get; set; } = new(); public List Licenses { get; set; } = new(); - // null = the source does not say; true/false = a provenance attestation was or was not found + // null when the source does not say public bool? HasProvenance { get; set; } public string? CheckedVersion { get; set; } } diff --git a/src/backend/PkgDiff.Shared/DTOs/SocialPostResult.cs b/src/backend/PkgDiff.Shared/DTOs/SocialPostResult.cs index 14ce190..1d53395 100644 --- a/src/backend/PkgDiff.Shared/DTOs/SocialPostResult.cs +++ b/src/backend/PkgDiff.Shared/DTOs/SocialPostResult.cs @@ -2,14 +2,12 @@ namespace PkgDiff.Shared.DTOs; public enum SocialPostOutcome { - // The platform confirmed the post (2xx) Succeeded, - // Definitely not posted: the request was never sent, or the platform answered with a 4xx + // Definitely not posted: never sent, or the platform answered with a 4xx Rejected, - // The request may have reached the platform but its fate is unknown (timeout, reset, 5xx); - // retrying could publish a duplicate + // May have reached the platform (timeout, reset, 5xx); retrying could publish a duplicate Unknown } diff --git a/src/backend/PkgDiff.Shared/ReleaseNotes/CompiledReleaseNotes.cs b/src/backend/PkgDiff.Shared/ReleaseNotes/CompiledReleaseNotes.cs index c5ffb84..5b9f872 100644 --- a/src/backend/PkgDiff.Shared/ReleaseNotes/CompiledReleaseNotes.cs +++ b/src/backend/PkgDiff.Shared/ReleaseNotes/CompiledReleaseNotes.cs @@ -3,9 +3,7 @@ namespace PkgDiff.Shared.ReleaseNotes; -// Notes pkgdiff compiles itself (from a GitHub milestone, when the maintainers publish no changelog) start -// with a notice line naming their source. The stored notes keep it, so copies and exports say it too; the -// page shows it as a warning apart from the notes. +// Notes pkgdiff compiles itself start with a notice line naming their source public static partial class CompiledReleaseNotes { private const string Marker = "Compiled by pkgdiff from the closed issues and pull requests of the "; @@ -13,7 +11,6 @@ public static partial class CompiledReleaseNotes public static string WithNotice(string notes, string label, string url) => $"> {Marker}[{label}]({url}); the maintainers publish no changelog for this version.\n\n{notes}"; - // The notes without the notice and the source it names; no source for notes the maintainers wrote public static (string Body, CompiledNotesSource? Source) Split(string? notes) { notes ??= string.Empty; diff --git a/src/backend/PkgDiff.Tests/Authorization/AdminAccountTakeoverTests.cs b/src/backend/PkgDiff.Tests/Authorization/AdminAccountTakeoverTests.cs index 7144ec3..5092a88 100644 --- a/src/backend/PkgDiff.Tests/Authorization/AdminAccountTakeoverTests.cs +++ b/src/backend/PkgDiff.Tests/Authorization/AdminAccountTakeoverTests.cs @@ -16,7 +16,6 @@ namespace PkgDiff.Tests.Authorization; -// CR-01: a non-Admin holding Users.Edit must not be able to take over an Admin account. public class AdminAccountTakeoverTests { private const string NewPassword = "N3w-Passw0rd!"; diff --git a/src/backend/PkgDiff.Tests/Authorization/AuthorizationTestHost.cs b/src/backend/PkgDiff.Tests/Authorization/AuthorizationTestHost.cs index b9a5017..2f88137 100644 --- a/src/backend/PkgDiff.Tests/Authorization/AuthorizationTestHost.cs +++ b/src/backend/PkgDiff.Tests/Authorization/AuthorizationTestHost.cs @@ -30,8 +30,7 @@ public static class AuthorizationTestHost Permissions.ScrapeLogs.View ]; - // Controller dependencies are intentionally not registered: authorization runs before controller - // activation, so a 500 from the activation failure means "authorization passed". + // Controllers are not registered: a 500 from their activation means authorization passed public static async Task StartAsync(Action? configure = null) { var builder = WebApplication.CreateBuilder(); diff --git a/src/backend/PkgDiff.Tests/Authorization/RoleMembershipEscalationTests.cs b/src/backend/PkgDiff.Tests/Authorization/RoleMembershipEscalationTests.cs index e908c1c..fb98518 100644 --- a/src/backend/PkgDiff.Tests/Authorization/RoleMembershipEscalationTests.cs +++ b/src/backend/PkgDiff.Tests/Authorization/RoleMembershipEscalationTests.cs @@ -22,8 +22,6 @@ namespace PkgDiff.Tests.Authorization; -// WR-01 (iteration 2): a non-Admin with Users.Create / Users.Edit must not obtain permissions they lack through -// role membership (for themselves or a new account) or by taking over a richer account. public class RoleMembershipEscalationTests { private const string SupportRole = "Support"; diff --git a/src/backend/PkgDiff.Tests/Authorization/RolePermissionEscalationTests.cs b/src/backend/PkgDiff.Tests/Authorization/RolePermissionEscalationTests.cs index 25d8357..08076ef 100644 --- a/src/backend/PkgDiff.Tests/Authorization/RolePermissionEscalationTests.cs +++ b/src/backend/PkgDiff.Tests/Authorization/RolePermissionEscalationTests.cs @@ -15,7 +15,6 @@ namespace PkgDiff.Tests.Authorization; -// WR-01: Roles.Edit must not let a non-Admin grant their own role permissions they do not hold. public class RolePermissionEscalationTests { private const string ManagerRole = "RoleManager"; diff --git a/src/backend/PkgDiff.Tests/Coalescing/WorkCancellationTests.cs b/src/backend/PkgDiff.Tests/Coalescing/WorkCancellationTests.cs index 5df91e6..77bdeee 100644 --- a/src/backend/PkgDiff.Tests/Coalescing/WorkCancellationTests.cs +++ b/src/backend/PkgDiff.Tests/Coalescing/WorkCancellationTests.cs @@ -9,7 +9,6 @@ public class WorkCancellationTests { private readonly WorkTrackerService _tracker = new(); - // The shared work of these tests: it reports the token it runs under and ends only when told to or when cancelled private sealed class SharedWork { private readonly TaskCompletionSource _result = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -30,7 +29,6 @@ public async Task RunAsync() } } - // What an endpoint does: the request's token is the ambient one while the call is made private Task CallAsync(SharedWork work, CancellationToken requestToken) { using (WorkCancellation.Use(requestToken)) diff --git a/src/backend/PkgDiff.Tests/Coalescing/WorkTrackerServiceTests.cs b/src/backend/PkgDiff.Tests/Coalescing/WorkTrackerServiceTests.cs index 4bc43cd..0884a69 100644 --- a/src/backend/PkgDiff.Tests/Coalescing/WorkTrackerServiceTests.cs +++ b/src/backend/PkgDiff.Tests/Coalescing/WorkTrackerServiceTests.cs @@ -22,8 +22,7 @@ public async Task Callers_with_the_same_key_share_one_run_and_its_result() return release.Task; }; - // A call has joined the run by the time it returns its task, so all fifty race for the same entry - // here and none of them can arrive after the run has ended + // A call has joined the run once it returns its task, so none can arrive after the run ends var callers = new Task[50]; Parallel.For(0, callers.Length, i => callers[i] = _tracker.RunCoalescedAsync("key", work)); Assert.True(_tracker.IsRunning("key")); @@ -74,7 +73,6 @@ public async Task A_caller_that_finishes_late_does_not_evict_a_newer_run_of_the_ var secondRun = _tracker.RunCoalescedAsync("key", () => second.Task); Assert.True(_tracker.IsRunning("key")); - // Another caller of the finished first run completes now; the second run must stay registered Assert.Equal(3, await _tracker.RunCoalescedAsync("other", () => Task.FromResult(3))); Assert.True(_tracker.IsRunning("key")); diff --git a/src/backend/PkgDiff.Tests/Content/AutoPilotApprovalTests.cs b/src/backend/PkgDiff.Tests/Content/AutoPilotApprovalTests.cs index ae8459e..a8890be 100644 --- a/src/backend/PkgDiff.Tests/Content/AutoPilotApprovalTests.cs +++ b/src/backend/PkgDiff.Tests/Content/AutoPilotApprovalTests.cs @@ -306,7 +306,6 @@ public async Task SocialJob_Retry_OnlyRetriesTheFailedPlatform() Assert.Equal(["LinkedIn"], _social.Calls); } - // WR-06: an unconfigured platform is skipped, not a failure that makes Hangfire retry [Fact] public async Task SocialJob_OnlyLinkedInConfigured_SucceedsWithoutTouchingTwitter() { @@ -402,7 +401,6 @@ public async Task SocialJob_BlogDeletedAfterPublish_DoesNotThrowOrPost() Assert.Empty(_unitOfWork.SocialMediaPosts.Items); } - // WR-06: the platform accepted the post but recording the outcome failed; the retry must not post again [Fact] public async Task SocialJob_RetryAfterUnrecordedSuccess_DoesNotPostTwice() { @@ -427,7 +425,6 @@ public async Task SocialJob_RetryAfterUnrecordedSuccess_DoesNotPostTwice() Assert.Equal(SocialMediaPostStatus.Pending, twitter.Status); } - // WR-03: a timeout or 5xx may already have published the post, so it stays Pending and is never retried [Fact] public async Task SocialJob_UnknownOutcome_StaysPending_DoesNotThrow_AndIsNotRepostedLater() { diff --git a/src/backend/PkgDiff.Tests/Content/ReleaseNotesNestingGuardTests.cs b/src/backend/PkgDiff.Tests/Content/ReleaseNotesNestingGuardTests.cs index e09c685..5480e5b 100644 --- a/src/backend/PkgDiff.Tests/Content/ReleaseNotesNestingGuardTests.cs +++ b/src/backend/PkgDiff.Tests/Content/ReleaseNotesNestingGuardTests.cs @@ -5,9 +5,7 @@ namespace PkgDiff.Tests.Content; -// Every input here is above the guard's limits (64 levels, 128 KiB) but at most 500 levels deep, an order of -// magnitude below the depth that overflows the stack (about 5,400 nested elements on a 1.5 MB stack). -// Crash-depth input must never be used in these tests: it would take down the test host. +// Inputs stay far below stack-overflow depth: a crash-depth input would take down the test host public class ReleaseNotesNestingGuardTests { private const string Marker = "SECRET-NOTE-TEXT"; diff --git a/src/backend/PkgDiff.Tests/Content/ReleaseNotesRendererTests.cs b/src/backend/PkgDiff.Tests/Content/ReleaseNotesRendererTests.cs index c9adac5..de229c7 100644 --- a/src/backend/PkgDiff.Tests/Content/ReleaseNotesRendererTests.cs +++ b/src/backend/PkgDiff.Tests/Content/ReleaseNotesRendererTests.cs @@ -21,8 +21,6 @@ public class ReleaseNotesRendererTests private readonly ReleaseNotesRenderer _renderer = new(); - // The reviewer's corpora (208 payloads): parser-divergence, link-obfuscation, backtick-pairing, - // fence, table, list and autolink attacks against the previous markdown sanitizer public static TheoryData Payloads() { var data = new TheoryData(); diff --git a/src/backend/PkgDiff.Tests/Content/SocialPostOutcomeTests.cs b/src/backend/PkgDiff.Tests/Content/SocialPostOutcomeTests.cs index 788e5d1..bf119c2 100644 --- a/src/backend/PkgDiff.Tests/Content/SocialPostOutcomeTests.cs +++ b/src/backend/PkgDiff.Tests/Content/SocialPostOutcomeTests.cs @@ -14,7 +14,6 @@ namespace PkgDiff.Tests.Content; -// WR-03: only a definite refusal may be retried; anything that may already have published the post is Unknown. public class SocialPostOutcomeTests { private readonly FakeUnitOfWork _uow = new(); diff --git a/src/backend/PkgDiff.Tests/Edge/EdgeTrustTests.cs b/src/backend/PkgDiff.Tests/Edge/EdgeTrustTests.cs index 07a74de..ae138c7 100644 --- a/src/backend/PkgDiff.Tests/Edge/EdgeTrustTests.cs +++ b/src/backend/PkgDiff.Tests/Edge/EdgeTrustTests.cs @@ -42,7 +42,7 @@ private static async Task SendRequestAsync( { ctx.Request.Path = path; configure(ctx); - // TestServer treats a null peer as a trusted proxy, so a missing peer would let spoofing tests pass for the wrong reason. + // TestServer trusts a null peer, so a missing peer would let spoofing tests pass wrongly Assert.NotNull(ctx.Connection.RemoteIpAddress); }, TestContext.Current.CancellationToken); } @@ -257,7 +257,6 @@ public async Task RateLimiting_AdminRoutes_HaveTheirOwnTierAndDoNotUseUpTheGener { await using var app = await StartRateLimitHostAsync(); - // More than the general tier allows in a minute, as the admin panel and the Hangfire dashboard do for (var i = 0; i < 100; i++) Assert.Equal(200, await SendFromVisitorAsync(app, i % 2 == 0 ? "/api/admin/stats" : "/hangfire/stats", signedIn: true)); @@ -296,7 +295,6 @@ public async Task RateLimiting_TiersAreReadFromConfiguration() Assert.Equal(429, await SendFromVisitorAsync(app, "/api/package/analyze")); } - // signedIn: the request carries a bearer token, which only gives an admin address the admin tier private static async Task SendFromVisitorAsync(WebApplication app, string path, bool signedIn = false) { var context = await SendRequestAsync(app, path, ctx => @@ -356,8 +354,7 @@ public async Task RateLimiting_PdfExportOverTheGeneralLimit_IsRejectedWithoutCou var app = builder.Build(); app.UseForwardedHeaders(); app.UseMiddleware(); - // No body write: TestServer.SendAsync returns once the response starts, which would race the - // middleware's post-response log write. An unstarted response completes after the pipeline. + // No body write: SendAsync returns once the response starts, racing the post-response log app.Run(_ => Task.CompletedTask); await app.StartAsync(TestContext.Current.CancellationToken); return (app, channel); diff --git a/src/backend/PkgDiff.Tests/Hangfire/HangfireDashboardTests.cs b/src/backend/PkgDiff.Tests/Hangfire/HangfireDashboardTests.cs index 015037f..796d497 100644 --- a/src/backend/PkgDiff.Tests/Hangfire/HangfireDashboardTests.cs +++ b/src/backend/PkgDiff.Tests/Hangfire/HangfireDashboardTests.cs @@ -41,8 +41,6 @@ private static HangfireDashboardAccess CreateAccess(IDataProtectionProvider? pro new(provider ?? new EphemeralDataProtectionProvider(), new MemoryDistributedCache(Options.Create(new MemoryDistributedCacheOptions()))); - // Ticket and cookie ------------------------------------------------------------------------ - [Fact] public async Task RedeemTicket_ValidTicket_ReturnsUserIdOnce() { @@ -93,8 +91,6 @@ public void CookieValue_RoundTripsAndRejectsGarbage() Assert.Null(access.ReadCookieValue(null)); } - // Per-request authorization ---------------------------------------------------------------- - private static DefaultHttpContext ContextWithCookie(string? cookieValue, ClaimsPrincipal? user = null) { var context = new DefaultHttpContext(); @@ -151,7 +147,6 @@ public async Task IsAuthorized_SubClaimOnlyAdminPrincipalStillActiveAdmin_True() Assert.True(await access.IsAuthorizedAsync(ContextWithCookie(null, principal), new FakeAdminRoleChecker(AdminUser))); } - // WR-04: a demoted, locked-out or deleted Admin keeps an unexpired JWT Admin role claim [Fact] public async Task IsAuthorized_NoCookieAdminPrincipalNoLongerAdmin_False() { @@ -171,8 +166,6 @@ public async Task IsAuthorized_AdminRoleClaimWithoutUserId_False() Assert.False(await access.IsAuthorizedAsync(ContextWithCookie(null, principal), new FakeAdminRoleChecker(AdminUser))); } - // Ticket middleware ------------------------------------------------------------------------ - private static async Task StartTicketHostAsync(HangfireDashboardAccess access) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions @@ -255,8 +248,6 @@ public async Task TicketMiddleware_NoTicket_PassesThrough() Assert.Equal("dashboard", await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken)); } - // Ticket endpoint -------------------------------------------------------------------------- - private static async Task StartEndpointHostAsync() { var builder = WebApplication.CreateBuilder(); diff --git a/src/backend/PkgDiff.Tests/Identity/AdminSeedingTests.cs b/src/backend/PkgDiff.Tests/Identity/AdminSeedingTests.cs index f801d9b..2426811 100644 --- a/src/backend/PkgDiff.Tests/Identity/AdminSeedingTests.cs +++ b/src/backend/PkgDiff.Tests/Identity/AdminSeedingTests.cs @@ -4,7 +4,6 @@ namespace PkgDiff.Tests.Identity; -// WR-02: the seeder bootstraps the first administrator only and never overrides admin decisions afterwards. public class AdminSeedingTests { private const string AdminUserName = "pkgdiff_admin"; diff --git a/src/backend/PkgDiff.Tests/Integration/HostIntegrationTests.cs b/src/backend/PkgDiff.Tests/Integration/HostIntegrationTests.cs index 933043d..1f680c4 100644 --- a/src/backend/PkgDiff.Tests/Integration/HostIntegrationTests.cs +++ b/src/backend/PkgDiff.Tests/Integration/HostIntegrationTests.cs @@ -9,7 +9,6 @@ namespace PkgDiff.Tests.Integration; -// One host for the whole class: starting it migrates a database, seeds it and installs the job storage public sealed class HostFixture : IAsyncLifetime { public PkgDiffHost? Host { get; private set; } @@ -30,8 +29,7 @@ public async ValueTask DisposeAsync() await Host.DisposeAsync(); - // The host must leave nothing of itself in the process: a Hangfire attribute asks Hangfire's - // static log provider for a logger when it is built, as every later test that reads one does + // Throws if Hangfire's static log provider still points at the disposed host _ = new global::Hangfire.AutomaticRetryAttribute(); } } @@ -112,7 +110,6 @@ public async Task Admin_endpoints_need_a_valid_token_and_the_permission_of_the_a Assert.True((await ReadJsonAsync(dashboard)).TryGetProperty("cachedPackages", out _)); Assert.Equal(HttpStatusCode.OK, (await admin.GetAsync("/api/admin/security/ip-statistics", TestContext.Current.CancellationToken)).StatusCode); - // The viewer may look at every list and change nothing using var viewer = await SignedInAsync("pkgdiff_viewer", PkgDiffHost.ViewerPassword); foreach (var path in new[] { "/api/admin/users", "/api/admin/roles", "/api/admin/packages", "/api/admin/registries", "/api/admin/settings", "/api/admin/scrape-logs", "/api/admin/security/request-logs" }) Assert.Equal(HttpStatusCode.OK, (await viewer.GetAsync(path, TestContext.Current.CancellationToken)).StatusCode); @@ -155,7 +152,6 @@ public async Task An_analysis_runs_end_to_end_and_records_the_visitor_behind_the Assert.Equal("1.1.0", package.LatestVersion); Assert.Equal(Published, package.LatestVersionDate); - // The notes are fetched in the background: queued once, not scraped inside the request Assert.Single(Host.Jobs.Enqueued, job => job == "IPackageAnalysisService.ScrapePackageReleaseNotesAsync"); var feed = await ReadJsonAsync(await client.GetAsync("/api/package/live-updates", TestContext.Current.CancellationToken)); @@ -163,7 +159,6 @@ public async Task An_analysis_runs_end_to_end_and_records_the_visitor_behind_the Assert.Equal("1.1.0", entry.GetProperty("version").GetString()); Assert.Equal(Published, entry.GetProperty("updatedAt").GetDateTime().ToUniversalTime()); - // The analysis shows on the Prometheus endpoint, which only a scraper inside the compose network may read var metrics = await (await ScrapeMetricsAsync(client, "172.28.0.9")).Content.ReadAsStringAsync(TestContext.Current.CancellationToken); Assert.Contains(metrics.Split('\n'), line => line.StartsWith("pkgdiff_analyses{", StringComparison.Ordinal) && line.Contains("registry=\"npm\"") && line.Contains("outcome=\"ok\"")); @@ -195,7 +190,6 @@ public async Task A_forwarded_address_from_an_untrusted_peer_is_ignored_and_bad_ Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); Assert.Contains("not found", (await ReadJsonAsync(response)).GetProperty("error").GetString()); - // The request counts against the peer that really sent it, not against the address it claimed var counters = (MemoryRateLimitCounterStore)Host.Services.GetRequiredService(); Assert.Equal(1, counters.GetCount("rl:1m:198.51.100.20")); Assert.Null(counters.GetCount("rl:1m:203.0.113.77")); diff --git a/src/backend/PkgDiff.Tests/Integration/PkgDiffHost.cs b/src/backend/PkgDiff.Tests/Integration/PkgDiffHost.cs index 8849bb6..29a20c2 100644 --- a/src/backend/PkgDiff.Tests/Integration/PkgDiffHost.cs +++ b/src/backend/PkgDiff.Tests/Integration/PkgDiffHost.cs @@ -18,9 +18,6 @@ namespace PkgDiff.Tests.Integration; -// The real application (Program.cs: every registration, the whole pipeline, migrations, seeding, -// the job schedule) on a PostgreSQL database of its own. Only the outside world is replaced: the -// npm registry and the job queue. public sealed class PkgDiffHost : WebApplicationFactory { public const string PeerHeader = "X-Test-Peer"; @@ -72,8 +69,7 @@ public async Task WithDatabaseAsync(Func> re public override async ValueTask DisposeAsync() { - // Hangfire keeps its log provider in a static. Left pointing at this host, every Hangfire object - // built later in the test run would ask the disposed logger factory for a logger and fail. + // Hangfire's log provider is static; left on this host, later Hangfire objects would fail global::Hangfire.Logging.LogProvider.SetCurrentLogProvider(null); await base.DisposeAsync(); @@ -83,8 +79,7 @@ public override async ValueTask DisposeAsync() await context.Database.EnsureDeletedAsync(); } - // TestServer has no socket, so a request has no peer address. A test names the peer it wants - // to be in a header, and this puts it on the connection before the application's own pipeline. + // TestServer has no socket, so a test names the peer address it wants in a header private sealed class PeerAddressFilter : IStartupFilter { public Action Configure(Action next) => app => diff --git a/src/backend/PkgDiff.Tests/Postgres/PostgresDatabase.cs b/src/backend/PkgDiff.Tests/Postgres/PostgresDatabase.cs index 66a95f3..9f023af 100644 --- a/src/backend/PkgDiff.Tests/Postgres/PostgresDatabase.cs +++ b/src/backend/PkgDiff.Tests/Postgres/PostgresDatabase.cs @@ -7,8 +7,6 @@ namespace PkgDiff.Tests.Postgres; -// A database of its own for one test, on the server PKGDIFF_TEST_POSTGRES points to, with every -// migration applied. The default suite has no server and skips these tests; CI provides one. public sealed class PostgresDatabase : IAsyncDisposable { public const string ConnectionVariable = "PKGDIFF_TEST_POSTGRES"; @@ -44,7 +42,6 @@ public static async Task WaitUntilReachableAsync() } } - // Migrated to the newest migration, or only as far as the one named public static async Task CreateAsync(string? targetMigration = null) { var connection = new NpgsqlConnectionStringBuilder(Environment.GetEnvironmentVariable(ConnectionVariable)) @@ -75,7 +72,6 @@ public static async Task CreateAsync(string? targetMigration = public ApplicationDbContext CreateContext() => new(_options); - // A fresh context each time, as each request gets its own public UnitOfWork CreateUnitOfWork() => new(CreateContext()); public async ValueTask DisposeAsync() diff --git a/src/backend/PkgDiff.Tests/Postgres/PostgresQueryTests.cs b/src/backend/PkgDiff.Tests/Postgres/PostgresQueryTests.cs index 7c58048..76de8d1 100644 --- a/src/backend/PkgDiff.Tests/Postgres/PostgresQueryTests.cs +++ b/src/backend/PkgDiff.Tests/Postgres/PostgresQueryTests.cs @@ -17,8 +17,6 @@ namespace PkgDiff.Tests.Postgres; -// The in-memory fakes run LINQ in the process and accept anything. These tests run the same code -// against PostgreSQL, where a query has to translate to SQL and a constraint is really enforced. public class PostgresQueryTests { private static readonly DateTime Now = DateTime.UtcNow; @@ -206,7 +204,6 @@ public async Task A_second_note_for_one_version_is_refused_and_written_onto_the_ await unitOfWork.CachedReleaseNotes.AddAsync(new CachedReleaseNote { CachedPackageId = packageId, Version = "1.3.0", ReleaseNotes = "dup", CachedAt = Now, ExpiresAt = Now }); await Assert.ThrowsAsync(() => unitOfWork.SaveChangesAsync()); - // The refused row is forgotten: the same unit of work can save again Assert.Equal(0, await unitOfWork.SaveChangesAsync()); var cache = new PackageCacheService( diff --git a/src/backend/PkgDiff.Tests/Postgres/UtcTimestampTests.cs b/src/backend/PkgDiff.Tests/Postgres/UtcTimestampTests.cs index 698c792..ba6b57b 100644 --- a/src/backend/PkgDiff.Tests/Postgres/UtcTimestampTests.cs +++ b/src/backend/PkgDiff.Tests/Postgres/UtcTimestampTests.cs @@ -64,7 +64,6 @@ public async Task Dates_of_any_kind_are_written_and_come_back_as_the_same_utc_in Assert.Equal((utc, DateTimeKind.Utc), (blocked.CreatedAt, blocked.CreatedAt.Kind)); Assert.Equal((utc, DateTimeKind.Utc), (blocked.ExpiresAt!.Value, blocked.ExpiresAt.Value.Kind)); - // A filter value without a kind means UTC as well, and the server's own clock compares correctly Assert.Equal(1, await reading.BlockedIps.CountAsync(b => b.ExpiresAt == moment, TestContext.Current.CancellationToken)); Assert.Equal(1, await reading.BlockedIps.CountAsync(b => b.ExpiresAt < DateTime.UtcNow, TestContext.Current.CancellationToken)); Assert.Equal(1, await reading.Registries.CountAsync(r => r.CreatedAt.Date == utc.Date, TestContext.Current.CancellationToken)); @@ -106,7 +105,6 @@ await context.Database.ExecuteSqlRawAsync( Assert.Equal(new DateTime(2026, 3, 1, 22, 30, 0, DateTimeKind.Utc), registry.CreatedAt); Assert.Equal(DateTimeKind.Utc, registry.CreatedAt.Kind); - // Back to the columns without a zone: the stored time is still the UTC one await reading.GetService().MigrateAsync(LastMigrationWithoutZones, TestContext.Current.CancellationToken); Assert.Equal("2026-03-01 22:30:00", await ReadSettingAsync( reading, """SELECT to_char("CreatedAt", 'YYYY-MM-DD HH24:MI:SS') FROM "Registries" """)); diff --git a/src/backend/PkgDiff.Tests/Secrets/DataProtectionKeyRingTests.cs b/src/backend/PkgDiff.Tests/Secrets/DataProtectionKeyRingTests.cs index d752dfe..aced52f 100644 --- a/src/backend/PkgDiff.Tests/Secrets/DataProtectionKeyRingTests.cs +++ b/src/backend/PkgDiff.Tests/Secrets/DataProtectionKeyRingTests.cs @@ -12,7 +12,6 @@ namespace PkgDiff.Tests.Secrets; -// WR-03: the Data Protection key ring stored in the database must be encrypted with key material kept outside it. public class DataProtectionKeyRingTests { private sealed class InMemoryXmlRepository : IXmlRepository diff --git a/src/backend/PkgDiff.Tests/Secrets/GitHubStartupProbeTests.cs b/src/backend/PkgDiff.Tests/Secrets/GitHubStartupProbeTests.cs index 4b28fc4..bf90673 100644 --- a/src/backend/PkgDiff.Tests/Secrets/GitHubStartupProbeTests.cs +++ b/src/backend/PkgDiff.Tests/Secrets/GitHubStartupProbeTests.cs @@ -8,7 +8,7 @@ namespace PkgDiff.Tests.Secrets; public class GitHubStartupProbeTests { - // Distinctive head and tail so any partial disclosure (old "abcd****wxyz" masking) is detectable. + // Distinctive head and tail so any partial disclosure is detectable private const string TokenHead = "Qz7K"; private const string TokenTail = "W3rY"; private const string Token = TokenHead + "fake0test0token0value0only" + TokenTail; diff --git a/src/backend/PkgDiff.Tests/Security/SsrfGuardTests.cs b/src/backend/PkgDiff.Tests/Security/SsrfGuardTests.cs index 88d0945..f74f662 100644 --- a/src/backend/PkgDiff.Tests/Security/SsrfGuardTests.cs +++ b/src/backend/PkgDiff.Tests/Security/SsrfGuardTests.cs @@ -38,7 +38,7 @@ public class SsrfGuardTests [InlineData("::ffff:10.0.0.1")] [InlineData("::ffff:169.254.169.254")] [InlineData("64:ff9b::a9fe:a9fe")] - // WR-05: IPv4-compatible, 6to4, deprecated site-local, Teredo and local-use NAT64 + // IPv4-compatible, 6to4, deprecated site-local, Teredo and local-use NAT64 [InlineData("::7f00:1")] [InlineData("::a9fe:a9fe")] [InlineData("::a00:1")] diff --git a/src/backend/PkgDiff.Tests/Smoke/RegistrySmokeTests.cs b/src/backend/PkgDiff.Tests/Smoke/RegistrySmokeTests.cs index a60078f..3ae2b26 100644 --- a/src/backend/PkgDiff.Tests/Smoke/RegistrySmokeTests.cs +++ b/src/backend/PkgDiff.Tests/Smoke/RegistrySmokeTests.cs @@ -22,7 +22,7 @@ public class RegistrySmokeTests private const string PubDevVersionsUrl = "https://pub.dev/packages/http/versions"; private const string PubDevChangelogUrl = "https://pub.dev/packages/http/changelog"; - // Mirrors the defaults seeded by DatabaseSeeder (a private local there, so it cannot be referenced) + // Mirrors the defaults seeded by DatabaseSeeder, which keeps them private private static readonly string[] SeededExcludePatterns = [ "dev", "ci", "build", "nightly", "snapshot", "local", "test", "rc", "alpha", "beta", @@ -113,7 +113,6 @@ public async Task Npm_ScopedPackage_SmokeAnalysis() Assert.Equal(new[] { "21.0.1", "21.1.0", "21.2.0" }, inRange); - // The registry lists 20.3.14 after 21.0.1 (publish order), so this also proves the sort Assert.Equal(new[] { "20.3.14", "21.0.0", "21.0.1", "21.1.0", "21.2.0", "21.2.1", "22.2.1" }, stable); Assert.Equal("@angular/core", metadata.PackageId); @@ -146,7 +145,6 @@ public async Task PyPI_SmokeAnalysis() Assert.Equal(new[] { "2.32.0", "2.32.3" }, inRange); - // PyPI lists 2.4.0 after 2.34.2 (text order) and 2.34.0.dev1 is a pre-release Assert.Equal(new[] { "2.4.0", "2.31.0", "2.32.0", "2.32.3", "2.34.2" }, stable); Assert.Equal("requests", metadata.PackageId); @@ -211,7 +209,7 @@ public async Task NuGet_VersionParsing_SmokeAnalysis() var filter = await CreateSeededVersionFilterAsync(); var patterns = (await filter.GetActiveExcludePatternsAsync()).ToList(); - // Parsed from the reversed list, so the order below comes from NuGetVersion and not from the fixture + // Reversed first, so the order comes from NuGetVersion and not from the fixture var parsed = listed.AsEnumerable().Reverse().Select(NuGetVersion.Parse).OrderBy(v => v).ToList(); var stable = parsed .Where(v => !v.IsPrerelease && !filter.ShouldExcludeVersion(v.ToString(), patterns)) @@ -226,11 +224,9 @@ public async Task NuGet_VersionParsing_SmokeAnalysis() Assert.Equal(new[] { "13.0.2", "13.0.3" }, inRange); - // The flat container lists versions in ascending SemVer order (4.5.9 < 4.5.10, 13.0.1-beta2 < 13.0.1) Assert.Equal(listed, parsed.Select(v => v.ToString())); Assert.Equal(listed.Where(v => !v.Contains('-')), stable.Select(v => v.ToString())); - // Every Newtonsoft.Json pre-release is a -betaN, which the seeded "beta" pattern drops on its own Assert.Contains("13.0.2-beta1", listed); Assert.Equal(stable, notExcluded); } diff --git a/src/backend/PkgDiff.Tests/TestDoubles/FakeUnitOfWork.cs b/src/backend/PkgDiff.Tests/TestDoubles/FakeUnitOfWork.cs index 6767d6d..77f8302 100644 --- a/src/backend/PkgDiff.Tests/TestDoubles/FakeUnitOfWork.cs +++ b/src/backend/PkgDiff.Tests/TestDoubles/FakeUnitOfWork.cs @@ -42,10 +42,10 @@ public sealed class FakeUnitOfWork : IUnitOfWork IRepository IUnitOfWork.PackageSecurityInfos => PackageSecurityInfos; IRepository IUnitOfWork.DependencyAudits => DependencyAudits; - // Simulates a database failure on the Nth SaveChangesAsync call (1-based) + // 1-based SaveChangesAsync call number public int? FailOnSaveChangesCall { get; set; } - // Simulates a unique-key conflict on the Nth SaveChangesAsync call (1-based) + // 1-based SaveChangesAsync call number public int? DuplicateOnSaveChangesCall { get; set; } public Task> ToListAsync(IQueryable query) => Task.FromResult(query.ToList()); diff --git a/src/backend/PkgDiff.Tests/TestDoubles/InMemoryIdentityStore.cs b/src/backend/PkgDiff.Tests/TestDoubles/InMemoryIdentityStore.cs index 1a1c71d..2e58558 100644 --- a/src/backend/PkgDiff.Tests/TestDoubles/InMemoryIdentityStore.cs +++ b/src/backend/PkgDiff.Tests/TestDoubles/InMemoryIdentityStore.cs @@ -5,7 +5,6 @@ namespace PkgDiff.Tests.TestDoubles; -// Just enough of the Identity store surface for UserManager/RoleManager role operations, without a database. public sealed class InMemoryIdentityStore : IUserRoleStore, IUserPasswordStore, @@ -72,7 +71,6 @@ public void Dispose() { } - // IUserStore public Task GetUserIdAsync(ApplicationUser user, CancellationToken cancellationToken) => Task.FromResult(user.Id); public Task GetUserNameAsync(ApplicationUser user, CancellationToken cancellationToken) => Task.FromResult(user.UserName); @@ -114,7 +112,6 @@ public Task DeleteAsync(ApplicationUser user, CancellationToken public Task FindByNameAsync(string normalizedUserName, CancellationToken cancellationToken) => Task.FromResult(_users.FirstOrDefault(u => u.NormalizedUserName == normalizedUserName)); - // IUserPasswordStore public Task SetPasswordHashAsync(ApplicationUser user, string? passwordHash, CancellationToken cancellationToken) { user.PasswordHash = passwordHash; @@ -127,7 +124,7 @@ public Task SetPasswordHashAsync(ApplicationUser user, string? passwordHash, Can public Task HasPasswordAsync(ApplicationUser user, CancellationToken cancellationToken) => Task.FromResult(user.PasswordHash is not null); - // IUserRoleStore (role names arrive normalized) + // Role names arrive normalized public Task AddToRoleAsync(ApplicationUser user, string roleName, CancellationToken cancellationToken) { _userRoles[user.Id].Add(roleName); @@ -149,7 +146,6 @@ public Task IsInRoleAsync(ApplicationUser user, string roleName, Cancellat public Task> GetUsersInRoleAsync(string roleName, CancellationToken cancellationToken) => Task.FromResult>(_users.Where(u => _userRoles[u.Id].Contains(roleName)).ToList()); - // IRoleStore public Task CreateAsync(IdentityRole role, CancellationToken cancellationToken) { _roles.Add(role); @@ -183,7 +179,6 @@ public Task SetNormalizedRoleNameAsync(IdentityRole role, string? normalizedName return Task.CompletedTask; } - // IRoleClaimStore public IReadOnlyCollection PermissionsOf(string roleName) => _roleClaims.TryGetValue(RoleNamed(roleName).Id, out var claims) ? claims.Where(c => c.Type == "Permission").Select(c => c.Value).ToList() diff --git a/src/backend/PkgDiff.Tests/TestDoubles/TestRegistries.cs b/src/backend/PkgDiff.Tests/TestDoubles/TestRegistries.cs index 5066b32..74d06e0 100644 --- a/src/backend/PkgDiff.Tests/TestDoubles/TestRegistries.cs +++ b/src/backend/PkgDiff.Tests/TestDoubles/TestRegistries.cs @@ -7,8 +7,6 @@ namespace PkgDiff.Tests.TestDoubles; -// The four real registry clients over whatever a test supplies. A test that only needs a registry's -// rules (its version scheme, its label, its range) supplies nothing. public static class TestRegistries { public static RegistryServiceFactory Create( @@ -22,7 +20,6 @@ public static RegistryServiceFactory Create( NuGetFirstScrapingStrategy? nugetFirst = null, PubDevScrapingStrategy? pubDevStrategy = null) { - // Metadata is cut by the exclude patterns, so a client always needs a filter; by default one without patterns versionFilter ??= new VersionFilterService( new FakeUnitOfWork(), new MemoryCache(new MemoryCacheOptions()), NullLogger.Instance); diff --git a/src/backend/PkgDiff.Tests/Versioning/RegistryVersionListingTests.cs b/src/backend/PkgDiff.Tests/Versioning/RegistryVersionListingTests.cs index 59f533e..336720c 100644 --- a/src/backend/PkgDiff.Tests/Versioning/RegistryVersionListingTests.cs +++ b/src/backend/PkgDiff.Tests/Versioning/RegistryVersionListingTests.cs @@ -15,7 +15,6 @@ public class RegistryVersionListingTests { private readonly FakeUnitOfWork _unitOfWork = new(); - // Listing versions needs neither pub.dev nor a scraping strategy private RegistryServiceFactory CreateFactory(IEnumerable listed) { var listing = new StubListing(listed); @@ -77,7 +76,6 @@ public async Task Exclude_patterns_apply_to_registries_other_than_nuget() Assert.Equal(new[] { "1.0.0", "1.1.0-canary.1", "1.1.0" }, all.Select(v => v.Value)); } - // The behaviour each registry has always had: NuGet leaves the starting version out, the others list it [Theory] [InlineData("nuget", new[] { "2.0.0", "1.5.0" })] [InlineData("npm", new[] { "2.0.0", "1.5.0", "1.0.0" })] diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index 15b643c..a648f34 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -1,50 +1,37 @@ -# Build stage: Node 22 LTS, pinned by tag and index digest. The Node binary of this stage also runs -# the render server in the final image, so what is built and tested is what runs. +# This stage's Node binary also runs the render server in the final image. FROM node:22.23.3-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402 AS build WORKDIR /app -# The lock file is mandatory: the install below takes exactly what it records COPY package.json package-lock.json ./ RUN npm ci COPY . . RUN npm run build -# Runtime stage: nginx stable line, pinned by tag and index digest. -# One image, two compose services: "frontend" runs nginx (the entrypoint below) and "ssr" runs the -# render server with Node, so the files nginx serves and the pages that refer to them are one build. -# Neither runs as root. +# One image, two services: "frontend" runs nginx and "ssr" runs the render server with Node. FROM nginx:1.30.6-alpine@sha256:970364228b1e4f932abc22d321932816782cca8a20b85a77552a7cfe9acfb3b5 AS final -# Pick up Alpine security fixes published after the nginx image was built, and install the C++ -# runtime the Node binary links against. -# The base digest is pinned, so Docker would reuse this layer from its build cache -# forever; bump the date (or pass --build-arg APK_UPGRADE_DATE=...) to run it again. +# With a pinned base this layer stays cached forever; bump the date to pick up security fixes. ARG APK_UPGRADE_DATE=2026-10-04 RUN apk upgrade --no-cache && apk add --no-cache libstdc++ COPY --from=build /usr/local/bin/node /usr/local/bin/node -# Clean default Nginx files RUN rm -rf /usr/share/nginx/html/* -# dist/Frontend holds browser/ (the files nginx serves) and server/ (the render server); the -# server bundle is self-contained and needs no node_modules +# The server bundle is self-contained and needs no node_modules COPY --from=build /app/dist/Frontend /app COPY nginx.conf /etc/nginx/conf.d/default.conf COPY nginx-security-headers.conf /etc/nginx/snippets/security-headers.conf -# nginx runs as the unprivileged "nginx" user from its first process on: the pid file moves to -# /tmp, the "user" directive (only a root master can honour it) goes, and the directory of its -# temporary files becomes its own. Port 80 needs no privilege inside a container. RUN sed -i -e '/^user /d' -e 's,^pid .*,pid /tmp/nginx.pid;,' /etc/nginx/nginx.conf \ && grep -q '^pid /tmp/nginx.pid;' /etc/nginx/nginx.conf \ && ! grep -q '^user ' /etc/nginx/nginx.conf \ && chown -R nginx:nginx /var/cache/nginx USER nginx -# The deploy script reads this to know that pages of this image must arrive rendered +# deploy/deploy.sh reads this to expect server-rendered pages LABEL dev.pkgdiff.rendering="server" EXPOSE 80 4000 diff --git a/src/frontend/nginx-security-headers.conf b/src/frontend/nginx-security-headers.conf index 762bd57..78a08a4 100644 --- a/src/frontend/nginx-security-headers.conf +++ b/src/frontend/nginx-security-headers.conf @@ -1,14 +1,8 @@ -# Sent with every response of this server: files, rendered pages and proxied API answers. -# nginx does not inherit add_header into a block that has an add_header of its own, so every such -# location includes this file again. HSTS is not here: it belongs to the layer that terminates TLS -# (the host nginx). +# Re-included by every location with its own add_header, since nginx does not inherit them. +# HSTS is set by the host nginx, which terminates TLS. add_header X-Content-Type-Options "nosniff" always; add_header X-Frame-Options "DENY" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; add_header Permissions-Policy "geolocation=(), microphone=(), camera=(), payment=()" always; -# No 'unsafe-eval': nothing in the application or the Hangfire dashboard evaluates strings. -# 'unsafe-inline' stays for scripts: the theme script in index.html, Angular's event replay and its -# stylesheet loader are inline. Cloudflare Web Analytics injects its beacon from static.cloudflareinsights.com -# and reports to cloudflareinsights.com. connect-src lists the API (same origin, the hub included), -# that beacon and Google Analytics; nothing else is called from a browser. +# Script 'unsafe-inline' is needed by the inline theme script and Angular's inline loaders. add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' https://www.googletagmanager.com https://static.cloudflareinsights.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: https:; font-src 'self' data: https://fonts.gstatic.com; connect-src 'self' https://cloudflareinsights.com https://www.googletagmanager.com https://www.google-analytics.com https://*.google-analytics.com https://*.analytics.google.com; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'" always; diff --git a/src/frontend/nginx.conf b/src/frontend/nginx.conf index 80a8392..56119c8 100644 --- a/src/frontend/nginx.conf +++ b/src/frontend/nginx.conf @@ -1,12 +1,9 @@ -# WebSocket Connection Upgrade Mapping map $http_upgrade $connection_upgrade { default upgrade; '' close; } -# TLS termination and domain redirects are handled by the host nginx. -# This container serves the built files, hands pages to the render server (compose service "ssr", -# the same image run with Node) and proxies the API and the hub, all over plain HTTP. +# Plain HTTP: TLS and domain redirects are handled by the host nginx. server { listen 80; server_name _; @@ -14,38 +11,29 @@ server { root /app/browser; charset utf-8; server_tokens off; - # The largest request is a blog post saved from the admin panel client_max_body_size 2m; include /etc/nginx/snippets/security-headers.conf; - # Client IP: trust X-Forwarded-For only when the peer is inside the compose - # subnet (the host nginx reaches us via docker-proxy on 172.28.0.1, the render - # server calls the API from inside it). The rightmost untrusted address becomes - # $remote_addr, which is then forwarded as the single X-Forwarded-For value - # (overwrite, never append). + # Trust X-Forwarded-For only from the compose subnet; it is overwritten, never appended, below. set_real_ip_from 172.28.0.0/16; real_ip_header X-Forwarded-For; real_ip_recursive on; - # Docker's DNS, asked when a page is requested instead of when nginx starts: nginx comes up and - # keeps serving the application even while the render server is missing or restarting. + # Resolve per request so nginx starts and serves even while the render server is down. resolver 127.0.0.11 valid=10s ipv6=off; - # Gzip Compression gzip on; gzip_vary on; gzip_min_length 1000; gzip_proxied any; gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml; - # A file of the build is served from disk; everything else is a page location / { try_files $uri @ssr; } - # Pages are rendered on the server. No X-Forwarded-Host/-Proto is passed on: the render server - # needs the visitor's address and nothing else, and it resolves the API inside this network. + # X-Forwarded-Host/-Proto are left out on purpose: the render server needs only the client IP. location @ssr { set $ssr_upstream http://ssr:4000; proxy_pass $ssr_upstream; @@ -56,11 +44,10 @@ server { proxy_connect_timeout 3s; proxy_read_timeout 20s; - # Without the render server the visitor still gets the application: the client-side shell, - # which renders the same page in the browser + # Without the render server, fall back to the client-side shell error_page 502 503 504 = /index.csr.html; - # A page names the files of one build; a stored copy would ask for files of an older one + # no-cache: a page references the hashed files of one build include /etc/nginx/snippets/security-headers.conf; add_header Cache-Control "no-cache" always; } @@ -71,7 +58,6 @@ server { add_header Cache-Control "no-cache" always; } - # Dynamic sitemap, generated by the backend location = /sitemap.xml { proxy_pass http://backend:8080/api/sitemap; proxy_set_header Host $host; @@ -80,11 +66,7 @@ server { proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; } - # API Proxy - # ^~ keeps an API address that ends like a file (a package named chart.js) away from the - # static-asset regex below. No URI part on proxy_pass: the request URI then goes on as the browser - # sent it. With one, nginx sends the decoded URI, and the %2F of a scoped npm name - # (/api/Package/npm/%40apollo%2Fserver) becomes a slash the API has no route for. + # ^~ beats the asset regexes; no URI on proxy_pass keeps an encoded %2F from being decoded. location ^~ /api/ { proxy_pass http://backend:8080; proxy_http_version 1.1; @@ -97,7 +79,6 @@ server { proxy_cache_bypass $http_upgrade; } - # SignalR Hub location ^~ /analysisHub { proxy_pass http://backend:8080/analysisHub; proxy_http_version 1.1; @@ -112,9 +93,7 @@ server { proxy_send_timeout 120s; } - # Hangfire dashboard (same-origin). ^~ stops the static-asset regex below - # from capturing the dashboard's .js/.css requests; no URI part on - # proxy_pass so the full /hangfire path reaches the backend. + # ^~ keeps the dashboard's .js/.css requests away from the asset regexes below. location ^~ /hangfire { proxy_pass http://backend:8080; proxy_http_version 1.1; @@ -124,10 +103,7 @@ server { proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; } - # Files of the build carry a hash of their content in their name: a changed file is a new - # address, so a stored copy never goes stale. - # In both blocks an address that only looks like a file (a package named chart.js) is a page; - # it then gets the headers of a page, not these. + # Hashed build files never go stale; a missing path that only looks like a file is a page. location ~ "^/[^/]+-[A-Z0-9]{8}\.(?:js|css)$" { try_files $uri @ssr; access_log off; @@ -135,7 +111,6 @@ server { add_header Cache-Control "public, max-age=31536000, immutable"; } - # Every other file keeps its name when it changes (icons, share images, the logo fonts) location ~* \.(?:ico|css|js|gif|jpe?g|png|woff2?|eot|ttf|svg|webp)$ { try_files $uri @ssr; access_log off; diff --git a/src/frontend/src/app/app.config.ts b/src/frontend/src/app/app.config.ts index 7578799..ed884a3 100644 --- a/src/frontend/src/app/app.config.ts +++ b/src/frontend/src/app/app.config.ts @@ -12,7 +12,6 @@ export const appConfig: ApplicationConfig = { providers: [ provideBrowserGlobalErrorListeners(), provideRouter(routes), - // What the server rendered is taken over, not rendered again; API answers travel with the page provideClientHydration(withEventReplay()), provideHttpClient(withFetch(), withInterceptors([forwardedForInterceptor, authInterceptor, errorInterceptor])), ] diff --git a/src/frontend/src/app/app.routes.server.ts b/src/frontend/src/app/app.routes.server.ts index 2349705..83c1fad 100644 --- a/src/frontend/src/app/app.routes.server.ts +++ b/src/frontend/src/app/app.routes.server.ts @@ -1,7 +1,6 @@ import { RenderMode, ServerRoute } from '@angular/ssr'; -// Public pages are rendered on the server for every request: their content comes from the API. -// The admin area needs a session that only the browser has, so it is sent as the client-side shell. +// The admin area needs a browser-only session, so it is sent as the client-side shell export const serverRoutes: ServerRoute[] = [ { path: '', renderMode: RenderMode.Server }, { path: 'packages', renderMode: RenderMode.Server }, @@ -19,6 +18,5 @@ export const serverRoutes: ServerRoute[] = [ { path: 'audit/:id', renderMode: RenderMode.Client }, { path: 'admin/login', renderMode: RenderMode.Client }, { path: 'admin/**', renderMode: RenderMode.Client }, - // Whatever is left is no page of the application { path: '**', renderMode: RenderMode.Server, status: 404 }, ]; diff --git a/src/frontend/src/app/app.routes.ts b/src/frontend/src/app/app.routes.ts index f1e34f4..41e2b2f 100644 --- a/src/frontend/src/app/app.routes.ts +++ b/src/frontend/src/app/app.routes.ts @@ -27,7 +27,6 @@ export const routes: Routes = [ loadComponent: () => import('./features/package/package').then(m => m.PackageComponent) }, { - // A scoped npm name typed with its slash, with a version: /package/npm/@scope/name/1.0.0 path: 'package/:registry/:scope/:name/:version', loadComponent: () => import('./features/package/package').then(m => m.PackageComponent) }, @@ -159,7 +158,6 @@ export const routes: Routes = [ loadComponent: () => import('./features/admin/social-media/admin-social-media').then(m => m.AdminSocialMediaComponent) }, { - // LinkedIn OAuth redirect target - uses the same component, reads ?code= in ngOnInit path: 'social-media/linkedin/callback', canActivate: [permissionGuard], data: { permission: 'Permissions.SocialMedia.Manage' }, diff --git a/src/frontend/src/app/app.ts b/src/frontend/src/app/app.ts index 2e071c0..5a61a47 100644 --- a/src/frontend/src/app/app.ts +++ b/src/frontend/src/app/app.ts @@ -24,13 +24,12 @@ export class App { constructor() { inject(AnalyticsService).init(); - // One hub connection and one feed for every page, in the browser only afterNextRender(() => { this.loadLiveUpdates(); this.signalr.startConnection().catch(() => undefined); }); - // Updates pushed while the connection was down are not replayed, so the feed is read again + // Updates pushed while disconnected are not replayed, so the feed is read again effect(() => { if (this.signalr.reconnected() > 0) { untracked(() => this.loadLiveUpdates()); diff --git a/src/frontend/src/app/core/config/app.config.ts b/src/frontend/src/app/core/config/app.config.ts index a5cb753..0c1f603 100644 --- a/src/frontend/src/app/core/config/app.config.ts +++ b/src/frontend/src/app/core/config/app.config.ts @@ -1,27 +1,23 @@ import { environment } from '../../../environments/environment'; export const APP_CONFIG = { - // App Information name: 'pkgdiff', shortName: 'pkgdiff', description: 'Compare and analyze package versions across NuGet, npm, PyPI and Pub registries. Track breaking changes, view detailed release notes, and understand version differences with our free online diff tool.', version: '1.0.0', - // Contact & Social contact: { email: 'info@pkgdiff.dev', github: 'https://github.com/pkgdiff-dev/pkgdiff', twitter: 'https://twitter.com/pkgdiff', }, - // Logo & Branding logo: { light: '/logo/logo-light.svg', dark: '/logo/logo-dark.svg', favicon: '/favicon/favicon.ico', }, - // Theme Configuration theme: { primary: '#3b82f6', secondary: '#8b5cf6', @@ -29,7 +25,6 @@ export const APP_CONFIG = { dark: '#1a1a1a', }, - // Footer Links footer: { sections: { product: { @@ -61,14 +56,12 @@ export const APP_CONFIG = { copyright: `© ${new Date().getFullYear()} pkgdiff. All rights reserved.`, }, - // Feature Flags features: { signalr: true, hangfire: true, analytics: false, }, - // API Configuration api: { baseUrl: environment.apiUrl, timeout: 30000, diff --git a/src/frontend/src/app/core/guards/guards.spec.ts b/src/frontend/src/app/core/guards/guards.spec.ts index 7280b31..7386cc0 100644 --- a/src/frontend/src/app/core/guards/guards.spec.ts +++ b/src/frontend/src/app/core/guards/guards.spec.ts @@ -46,7 +46,6 @@ describe('route guards', () => { auth.hasPermission.mockReturnValue(false); expect(urlOf(run(() => permissionGuard(users, state)))).toBe('/admin/dashboard'); - // A route that names no permission is open to every signed-in user expect(run(() => permissionGuard(routeWith({}), state))).toBe(true); }); diff --git a/src/frontend/src/app/core/guards/guest.guard.ts b/src/frontend/src/app/core/guards/guest.guard.ts index 409d579..d73abaa 100644 --- a/src/frontend/src/app/core/guards/guest.guard.ts +++ b/src/frontend/src/app/core/guards/guest.guard.ts @@ -7,10 +7,8 @@ export const guestGuard: CanActivateFn = () => { const router = inject(Router); if (auth.hasValidSession()) { - // Already logged in, redirect to dashboard return router.createUrlTree(['/admin/dashboard']); } - // Not logged in, allow access to login page return true; }; diff --git a/src/frontend/src/app/core/guards/permission.guard.ts b/src/frontend/src/app/core/guards/permission.guard.ts index ae336a1..8c930f6 100644 --- a/src/frontend/src/app/core/guards/permission.guard.ts +++ b/src/frontend/src/app/core/guards/permission.guard.ts @@ -2,7 +2,7 @@ import { CanActivateFn, Router } from '@angular/router'; import { inject } from '@angular/core'; import { AuthService } from '../services/auth.service'; -// A route names the permission it needs in data.permission; a route without one is open to every signed-in user +// A route without data.permission is open to every signed-in user export const permissionGuard: CanActivateFn = (route) => { const auth = inject(AuthService); const router = inject(Router); diff --git a/src/frontend/src/app/core/interceptors/auth.interceptor.ts b/src/frontend/src/app/core/interceptors/auth.interceptor.ts index f3373b8..dfd408b 100644 --- a/src/frontend/src/app/core/interceptors/auth.interceptor.ts +++ b/src/frontend/src/app/core/interceptors/auth.interceptor.ts @@ -15,8 +15,7 @@ export const authInterceptor: HttpInterceptorFn = (req, next) => { return next(req.clone({ setHeaders: { Authorization: `Bearer ${token}` } })).pipe( catchError((error: unknown) => { - // Only a request that carried the token can say the token is no longer accepted; - // a failed login attempt carries none and stays on the login page + // Only a request that carried the token can show it is no longer accepted if (error instanceof HttpErrorResponse && error.status === 401) { auth.handleUnauthorized(); } diff --git a/src/frontend/src/app/core/interceptors/error.interceptor.ts b/src/frontend/src/app/core/interceptors/error.interceptor.ts index 1373f08..21f918f 100644 --- a/src/frontend/src/app/core/interceptors/error.interceptor.ts +++ b/src/frontend/src/app/core/interceptors/error.interceptor.ts @@ -4,8 +4,7 @@ import { isPlatformBrowser } from '@angular/common'; import { catchError, throwError } from 'rxjs'; import { ToastService } from '../services/toast.service'; -// Says what no caller can say better: the server cannot be reached, or it is refusing requests for now. -// Every other failure stays with the caller, which knows what was being loaded and resets its own state. +// Reports only unreachable or throttled responses; every other failure stays with the caller export const errorInterceptor: HttpInterceptorFn = (req, next) => { const toast = inject(ToastService); const isBrowser = isPlatformBrowser(inject(PLATFORM_ID)); diff --git a/src/frontend/src/app/core/interceptors/forwarded-for.interceptor.ts b/src/frontend/src/app/core/interceptors/forwarded-for.interceptor.ts index 9828427..117d246 100644 --- a/src/frontend/src/app/core/interceptors/forwarded-for.interceptor.ts +++ b/src/frontend/src/app/core/interceptors/forwarded-for.interceptor.ts @@ -2,9 +2,7 @@ import { HttpInterceptorFn } from '@angular/common/http'; import { REQUEST, inject } from '@angular/core'; import { environment } from '../../../environments/environment'; -// During a server render the API is called by the render server, on behalf of the visitor whose page -// it is. Passing the visitor's address along keeps rate limits and request logs per visitor instead of -// charging every render to the server. In the browser there is no incoming request and nothing happens. +// Keeps rate limits and request logs per visitor when the render server calls the API export const forwardedForInterceptor: HttpInterceptorFn = (req, next) => { const visitor = inject(REQUEST, { optional: true })?.headers.get('x-forwarded-for'); diff --git a/src/frontend/src/app/core/interceptors/interceptors.spec.ts b/src/frontend/src/app/core/interceptors/interceptors.spec.ts index af0bcbf..5306065 100644 --- a/src/frontend/src/app/core/interceptors/interceptors.spec.ts +++ b/src/frontend/src/app/core/interceptors/interceptors.spec.ts @@ -47,7 +47,6 @@ describe('HTTP interceptors', () => { backend.expectOne(api('admin/users')).flush({ error: 'expired' }, { status: 401, statusText: 'Unauthorized' }); expect(auth.handleUnauthorized).toHaveBeenCalledTimes(1); - // The caller still gets the failure and resets its own state expect(failures).toEqual([401]); }); diff --git a/src/frontend/src/app/core/models/package.models.ts b/src/frontend/src/app/core/models/package.models.ts index 6e03460..3c0c0b2 100644 --- a/src/frontend/src/app/core/models/package.models.ts +++ b/src/frontend/src/app/core/models/package.models.ts @@ -2,7 +2,7 @@ import { PackageTrust } from './security.models'; export interface AnalyzeRequest { packageId: string; - registry: string; // Selected registry (nuget, npm, etc.) + registry: string; currentVersion: string; targetVersion: string; includePrerelease: boolean; @@ -66,7 +66,6 @@ export interface VersionInfo { isPrerelease: boolean; } -// Where notes pkgdiff compiled itself come from (a GitHub milestone): the maintainers did not write them export interface CompiledNotesSource { label: string; url: string | null; diff --git a/src/frontend/src/app/core/services/auth.service.spec.ts b/src/frontend/src/app/core/services/auth.service.spec.ts index 7023cb9..3025ddf 100644 --- a/src/frontend/src/app/core/services/auth.service.spec.ts +++ b/src/frontend/src/app/core/services/auth.service.spec.ts @@ -104,7 +104,6 @@ describe('AuthService', () => { expect(auth.isAuthenticated()).toBe(false); expect(router.navigate).toHaveBeenCalledWith(['/admin/login']); - // The old expiry timer is gone with the session vi.advanceTimersByTime(120_000); expect(router.navigate).toHaveBeenCalledTimes(1); }); diff --git a/src/frontend/src/app/core/services/auth.service.ts b/src/frontend/src/app/core/services/auth.service.ts index 3a98063..65ca4c1 100644 --- a/src/frontend/src/app/core/services/auth.service.ts +++ b/src/frontend/src/app/core/services/auth.service.ts @@ -28,7 +28,7 @@ export class AuthService { this.armExpiry(stored.expiresAt); } - // A timer does not run while the machine sleeps; the session is checked again when the tab comes back + // Timers stall while the machine sleeps, so the session is rechecked when the tab returns if (this.isBrowser) { document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'visible') this.hasValidSession(); @@ -58,12 +58,10 @@ export class AuthService { hasPermission(permission: string): boolean { const user = this.currentUser(); if (!user) return false; - // Admin role has all permissions by default if (user.role === 'Admin') return true; return (user.permissions || []).includes(permission); } - // Ends the session if its time is up, however late the timer is hasValidSession(): boolean { const user = this.currentUser(); if (!user) return false; @@ -79,7 +77,6 @@ export class AuthService { this.router.navigate(['/admin/login']); } - // The server refused the token: it expired, the user was removed or the signing key changed handleUnauthorized(): void { this.endSession(); } @@ -88,7 +85,6 @@ export class AuthService { return this.hasValidSession() ? this.currentUser()!.token : null; } - // A visitor on a public page stays where they are; only the admin area leads to the login page private endSession(): void { this.clear(); if (this.router.url.startsWith('/admin')) { diff --git a/src/frontend/src/app/core/services/browser-storage.service.ts b/src/frontend/src/app/core/services/browser-storage.service.ts index 5b18f36..1a26e37 100644 --- a/src/frontend/src/app/core/services/browser-storage.service.ts +++ b/src/frontend/src/app/core/services/browser-storage.service.ts @@ -1,7 +1,6 @@ import { Injectable, PLATFORM_ID, inject } from '@angular/core'; import { isPlatformBrowser } from '@angular/common'; -// localStorage that is simply empty where there is none: on the server, and in a browser that blocks it @Injectable({ providedIn: 'root' }) export class BrowserStorage { private readonly storage = isPlatformBrowser(inject(PLATFORM_ID)) ? this.open() : null; diff --git a/src/frontend/src/app/core/services/export.service.spec.ts b/src/frontend/src/app/core/services/export.service.spec.ts index 23a6ad2..1cdde0f 100644 --- a/src/frontend/src/app/core/services/export.service.spec.ts +++ b/src/frontend/src/app/core/services/export.service.spec.ts @@ -203,10 +203,8 @@ describe('ExportService', () => { { category: 'Bug Fixes', text: versions[0].entries[1].text }, ]); expect(parsed.versions[1].release_notes).toBe(versions[1].releaseNotes); - // TOML has no null: a version without a publish date or source omits those keys expect(parsed.versions[1]).not.toHaveProperty('published_date'); expect(parsed.versions[1]).not.toHaveProperty('release_notes_source'); - // Counts are TOML integers, not floats (2, never 2.0) expect(text).toContain('diff.total_versions = 2\n'); expect(text).toContain('summary.breaking_changes = 1\n'); expect(text).toContain('summary.new_features = 0\n'); diff --git a/src/frontend/src/app/core/services/export.service.ts b/src/frontend/src/app/core/services/export.service.ts index f1e3285..9acc2b1 100644 --- a/src/frontend/src/app/core/services/export.service.ts +++ b/src/frontend/src/app/core/services/export.service.ts @@ -15,9 +15,6 @@ export class ExportService { private api = inject(ApiService); private toast = inject(ToastService); - /** - * Export package metadata to specified format - */ async exportReleaseNotes( metadata: PackageMetadata, versions: VersionInfo[], @@ -53,9 +50,6 @@ export class ExportService { } } - /** - * Export diff results to specified format - */ async exportDiff( diff: AnalysisResult, registry: string, @@ -90,8 +84,6 @@ export class ExportService { } } - // ===== JSON Export ===== - private exportToJson(metadata: PackageMetadata, versions: VersionInfo[]): Blob { const json = JSON.stringify({ ...metadata, @@ -117,8 +109,6 @@ export class ExportService { }; } - // ===== Markdown Export ===== - private exportToMarkdown(metadata: PackageMetadata, versions: VersionInfo[]): Blob { let md = `# ${metadata.packageId}\n\n`; md += `**Latest Version:** ${metadata.latestVersion}\n`; @@ -198,8 +188,6 @@ export class ExportService { return new Blob([md], { type: 'text/markdown' }); } - // ===== TOML Export ===== - private exportToToml(metadata: PackageMetadata, versions: VersionInfo[]): Blob { const tomlObject = { package: { @@ -261,14 +249,13 @@ export class ExportService { return new Blob([this.toToml(tomlObject)], { type: 'application/toml' }); } - // Without the newline option j-toml returns an array of lines, which a Blob joins with commas + // Without the newline option j-toml returns lines, which a Blob would join with commas private toToml(table: object): string { const rootTable = this.toTomlValue(table) as Parameters[0]; return tomlStringify(rootTable, { newline: '\n' }); } - // TOML has no null: j-toml throws on null/undefined, so those keys are left out. - // j-toml writes a JS number as a float (2 becomes 2.0) and a bigint as an integer; the counts are integers. + // j-toml throws on null and writes numbers as floats: drop nulls, write ints as bigint private toTomlValue(value: unknown): unknown { if (typeof value === 'number' && Number.isInteger(value)) { return BigInt(value); @@ -286,8 +273,6 @@ export class ExportService { return value; } - // ===== PDF Export (via Backend) ===== - private async exportToPdfViaBackend(registry: string, packageId: string): Promise { try { const blob = await firstValueFrom(this.api.getBlob(`Package/${registry}/${encodeURIComponent(packageId)}/export/pdf`)); @@ -315,13 +300,11 @@ export class ExportService { } private reportPdfFailure(error: unknown): void { - // The error interceptor has already said these two + // The error interceptor already reports these if (error instanceof HttpErrorResponse && (error.status === 0 || error.status === 429)) return; this.toast.error('PDF export failed. Please try again.', 3000); } - // ===== Helper Methods ===== - private packageSecurityMarkdown(trust: PackageTrust | null | undefined, latestVersion: string, versions: string[]): string { if (!trust || trust.status !== 'ready') return ''; let md = `\n## Security\n\n`; diff --git a/src/frontend/src/app/core/services/http-status.service.ts b/src/frontend/src/app/core/services/http-status.service.ts index 7430d40..00ccae6 100644 --- a/src/frontend/src/app/core/services/http-status.service.ts +++ b/src/frontend/src/app/core/services/http-status.service.ts @@ -1,7 +1,6 @@ import { Injectable, RESPONSE_INIT, inject } from '@angular/core'; -// The HTTP status of the page while it is rendered on the server, so a missing package or post is a -// real 404 and not a 200 that says "not found". In the browser there is no response to set. +// Lets a server render answer a missing page with a real 404 instead of a 200 @Injectable({ providedIn: 'root' }) export class HttpStatusService { private response = inject(RESPONSE_INIT, { optional: true }); diff --git a/src/frontend/src/app/core/services/live-updates.service.ts b/src/frontend/src/app/core/services/live-updates.service.ts index e9b68e4..22df7ed 100644 --- a/src/frontend/src/app/core/services/live-updates.service.ts +++ b/src/frontend/src/app/core/services/live-updates.service.ts @@ -3,7 +3,6 @@ import { LiveUpdate } from '../models/package.models'; import { mergeLiveUpdate } from '../utils/live-updates.utils'; import { SignalrService } from './signalr.service'; -// App-wide state of the Live Updates feed: the first page from the API, kept current by the hub @Injectable({ providedIn: 'root' }) export class LiveUpdatesService { private signalr = inject(SignalrService); diff --git a/src/frontend/src/app/core/services/seo.service.ts b/src/frontend/src/app/core/services/seo.service.ts index 07a7734..4107a55 100644 --- a/src/frontend/src/app/core/services/seo.service.ts +++ b/src/frontend/src/app/core/services/seo.service.ts @@ -29,7 +29,6 @@ export class SeoService { private readonly siteName = 'pkgdiff'; private readonly baseUrl = 'https://pkgdiff.dev'; - // 1200x630 private readonly defaultOgImage = '/og-image/home.png'; updateMeta(config: SeoConfig): void { @@ -47,7 +46,6 @@ export class SeoService { this.updateCanonicalUrl(`${this.baseUrl}${config.url}`); } - // Open Graph: Slack, Reddit and Discord read these as well this.meta.updateTag({ property: 'og:title', content: config.title }); this.meta.updateTag({ property: 'og:description', content: config.description }); @@ -60,7 +58,6 @@ export class SeoService { this.meta.updateTag({ property: 'og:type', content: config.type }); } - // Always an absolute URL, with the site image as the fallback const rawImageUrl = config.image || this.defaultOgImage; const imageUrl = rawImageUrl.startsWith('http') ? rawImageUrl : `${this.baseUrl}${rawImageUrl}`; this.meta.updateTag({ property: 'og:image', content: imageUrl }); @@ -78,7 +75,6 @@ export class SeoService { this.meta.updateTag({ name: 'theme-color', content: '#0a0a0a' }); } - // For a page that does not exist: a title, and a request not to index it setNotFound(title: string, description: string): void { this.title.setTitle(title); this.meta.updateTag({ name: 'description', content: description }); @@ -96,7 +92,6 @@ export class SeoService { this.removeJsonLd('page'); } - // One breadcrumb per page: setting it again replaces the one that is there setBreadcrumb(items: BreadcrumbItem[]): void { this.setJsonLd('breadcrumb', { '@context': 'https://schema.org', @@ -114,13 +109,11 @@ export class SeoService { this.removeJsonLd('breadcrumb'); } - // What a page added for itself and the next page must not inherit clearPageData(): void { this.removeStructuredData(); this.removeBreadcrumb(); } - // Before the package is loaded only its name is known setPackagePlaceholder(packageId: string, registry: string, version?: string): void { const registryName = registry.toUpperCase(); this.updateMeta({ @@ -147,7 +140,6 @@ export class SeoService { } description += `. View ${versionCount} version${versionCount !== 1 ? 's' : ''} and compare release notes.`; - // Search results cut a description at about 160 characters if (description.length > 160) { description = description.substring(0, 157) + '...'; } diff --git a/src/frontend/src/app/core/services/signalr.service.ts b/src/frontend/src/app/core/services/signalr.service.ts index dbcf4d8..7448900 100644 --- a/src/frontend/src/app/core/services/signalr.service.ts +++ b/src/frontend/src/app/core/services/signalr.service.ts @@ -6,7 +6,6 @@ import { LiveUpdate } from '../models/package.models'; const MAX_RETRY_DELAY_MS = 30000; const retryDelay = (attempt: number) => Math.min(MAX_RETRY_DELAY_MS, 1000 * 2 ** Math.min(attempt, 5)); -// The part of a SignalR connection this service uses export interface HubConnectionLike { connectionId: string | null; on(event: string, handler: (...args: T) => void): void; @@ -16,7 +15,7 @@ export interface HubConnectionLike { invoke(method: string): Promise; } -// The SignalR client is loaded only when a connection is opened, which never happens on the server +// Loaded lazily on first connect, which never happens on the server export const HUB_CONNECTION_FACTORY = new InjectionToken<() => Promise>('HUB_CONNECTION_FACTORY', { providedIn: 'root', factory: () => async () => { @@ -29,8 +28,7 @@ export const HUB_CONNECTION_FACTORY = new InjectionToken<() => Promise(null); progressToken = signal(null); liveUpdate = signal(null); - // Counts reconnects: whatever was pushed while the connection was down has to be fetched again reconnected = signal(0); packageUpdated = signal(null); packageUpdateStarted = signal(null); - // New security data of a package was stored securityUpdated = signal(null); backgroundProgress = signal(null); - // The number of the newest package event: a page that opens later can tell old events from new ones + // Lets a page that opens later tell old events from new ones get lastSequence(): number { return this.eventSequence; } - // One connection for the whole app: every caller gets the same start startConnection(): Promise { if (!this.isBrowser) { return Promise.resolve(); @@ -80,7 +75,6 @@ export class SignalrService { return this.started; } - // Progress belongs to one package page; the next page starts without the last one's messages resetProgress(): void { this.progressMessage.set(''); this.backgroundProgress.set(null); @@ -143,14 +137,13 @@ export class SignalrService { await this.refreshProgressToken(); } catch (err) { console.error('Error while starting SignalR connection: ' + err); - // Automatic reconnect only covers a connection that was up once, so the first start retries here + // Auto-reconnect only covers a connection that was up once this.started = null; setTimeout(() => this.startConnection().catch(() => undefined), retryDelay(this.startAttempts++)); throw err; } } - // Without a token the analysis still runs; it just gets no progress messages private refreshProgressToken(): Promise { const connection = this.hubConnection; if (!connection) { diff --git a/src/frontend/src/app/core/services/theme.service.ts b/src/frontend/src/app/core/services/theme.service.ts index 415f642..01f61b1 100644 --- a/src/frontend/src/app/core/services/theme.service.ts +++ b/src/frontend/src/app/core/services/theme.service.ts @@ -14,8 +14,7 @@ export class ThemeService { readonly theme = signal(this.getInitialTheme()); constructor() { - // The server does not know the visitor's theme and leaves the class to the inline script in - // index.html, which sets it before the first paint + // On the server the inline script in index.html sets the theme class before first paint if (this.isBrowser) { this.applyTheme(this.theme()); } diff --git a/src/frontend/src/app/core/utils/http-error.utils.ts b/src/frontend/src/app/core/utils/http-error.utils.ts index b4d9205..0d83596 100644 --- a/src/frontend/src/app/core/utils/http-error.utils.ts +++ b/src/frontend/src/app/core/utils/http-error.utils.ts @@ -2,19 +2,17 @@ import { HttpErrorResponse } from '@angular/common/http'; export type LoadFailure = 'not-found' | 'error'; -// A missing item and a failed request are different things to show export function loadFailureOf(error: unknown): LoadFailure { return error instanceof HttpErrorResponse && error.status === 404 ? 'not-found' : 'error'; } -// Why the API turned a request down (a taken name, a weak password); any other failure keeps the caller's words export function rejectionMessage(error: unknown, fallback: string): string { return error instanceof HttpErrorResponse && error.status === 400 ? apiErrorMessage(error, fallback) : fallback; } -// The message the API sent with a failure: `error` on its own responses, `detail` on ProblemDetails +// `error` on the API's own responses, `detail` on ProblemDetails export function apiErrorMessage(error: unknown, fallback: string): string { - // Status 0: no answer at all; what the browser says about it ("Failed to fetch") means nothing to a visitor + // Status 0 means no response; the browser's own text means nothing to a visitor if (!(error instanceof HttpErrorResponse) || error.status === 0) return fallback; const body: unknown = error.error; diff --git a/src/frontend/src/app/core/utils/install-command.utils.ts b/src/frontend/src/app/core/utils/install-command.utils.ts index 75b1fe1..4824393 100644 --- a/src/frontend/src/app/core/utils/install-command.utils.ts +++ b/src/frontend/src/app/core/utils/install-command.utils.ts @@ -3,7 +3,6 @@ export interface InstallSnippet { code: string; } -// What a developer pastes to pin this exact version: the CLI command first, then the manifest line export function installSnippets(registry: string, packageId: string, version: string): InstallSnippet[] { switch (registry.toLowerCase()) { case 'npm': diff --git a/src/frontend/src/app/core/utils/live-updates.utils.ts b/src/frontend/src/app/core/utils/live-updates.utils.ts index 9f35a27..ce96c23 100644 --- a/src/frontend/src/app/core/utils/live-updates.utils.ts +++ b/src/frontend/src/app/core/utils/live-updates.utils.ts @@ -3,7 +3,6 @@ import { LiveUpdate } from '../models/package.models'; const sameEntry = (a: LiveUpdate, b: LiveUpdate) => a.packageId === b.packageId && a.registry === b.registry; -// A package appears once: a pushed update replaces its entry and the list stays newest first export function mergeLiveUpdate(items: LiveUpdate[], update: LiveUpdate, limit: number): LiveUpdate[] { return [update, ...items.filter((item) => !sameEntry(item, update))] .sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt)) diff --git a/src/frontend/src/app/core/utils/markdown.utils.ts b/src/frontend/src/app/core/utils/markdown.utils.ts index b046abd..a5f0de4 100644 --- a/src/frontend/src/app/core/utils/markdown.utils.ts +++ b/src/frontend/src/app/core/utils/markdown.utils.ts @@ -1,19 +1,17 @@ -// Plain text of a Markdown fragment, for excerpts and meta descriptions export function stripMarkdown(text: string): string { return text - .replace(/#{1,6}\s+/g, '') // headings - .replace(/\*\*(.+?)\*\*/g, '$1') // bold - .replace(/\*(.+?)\*/g, '$1') // italic - .replace(/\[(.+?)\]\(.+?\)/g, '$1') // links - .replace(/`{1,3}[^`]*`{1,3}/g, '') // inline/block code - .replace(/^\s*[-*+]\s+/gm, '') // list items - .replace(/^\s*\d+\.\s+/gm, '') // ordered list - .replace(/^[-_*]{3,}$/gm, '') // horizontal rules - .replace(/\s+/g, ' ') // one space where lines and removed parts met + .replace(/#{1,6}\s+/g, '') + .replace(/\*\*(.+?)\*\*/g, '$1') + .replace(/\*(.+?)\*/g, '$1') + .replace(/\[(.+?)\]\(.+?\)/g, '$1') + .replace(/`{1,3}[^`]*`{1,3}/g, '') + .replace(/^\s*[-*+]\s+/gm, '') + .replace(/^\s*\d+\.\s+/gm, '') + .replace(/^[-_*]{3,}$/gm, '') + .replace(/\s+/g, ' ') .trim(); } -// At most `length` characters, cut at a word where possible export function excerptOf(text: string, length = 160): string { const plain = stripMarkdown(text); if (plain.length <= length) return plain; diff --git a/src/frontend/src/app/core/utils/trust.utils.ts b/src/frontend/src/app/core/utils/trust.utils.ts index f5cc55d..5dce572 100644 --- a/src/frontend/src/app/core/utils/trust.utils.ts +++ b/src/frontend/src/app/core/utils/trust.utils.ts @@ -9,7 +9,6 @@ export function advisoriesOf(trust: PackageTrust | null | undefined, version: st return ids.map(id => byId.get(id) ?? unknownAdvisory(id)); } -// Whether OSV was asked about this version at all export function isChecked(trust: PackageTrust | null | undefined, version: string): boolean { return !!trust && trust.status === 'ready' && version in trust.versionAdvisories; } diff --git a/src/frontend/src/app/features/admin/blogs/admin-blogs.html b/src/frontend/src/app/features/admin/blogs/admin-blogs.html index 5585267..4f54630 100644 --- a/src/frontend/src/app/features/admin/blogs/admin-blogs.html +++ b/src/frontend/src/app/features/admin/blogs/admin-blogs.html @@ -2,7 +2,6 @@
-

AutoPilot Blogs

@@ -19,7 +18,6 @@

AutoPilot Blogs

-
@if (loading()) {
diff --git a/src/frontend/src/app/features/admin/blogs/admin-blogs.ts b/src/frontend/src/app/features/admin/blogs/admin-blogs.ts index e1c0a50..3c60f49 100644 --- a/src/frontend/src/app/features/admin/blogs/admin-blogs.ts +++ b/src/frontend/src/app/features/admin/blogs/admin-blogs.ts @@ -57,7 +57,7 @@ export class AdminBlogsComponent { previewHtml = signal(''); previewLoading = signal(false); - // Escape or a click beside the dialog must not throw away an edited post; Cancel still closes it + // Escape or a backdrop click must not discard an edited post; Cancel still closes it editDirty = computed(() => { const blog = this.editingBlog(); return blog !== null && (this.editTitle() !== blog.title || this.editContent() !== blog.content); @@ -92,14 +92,12 @@ export class AdminBlogsComponent { this.editingBlog.set(null); } - // The preview is rendered by the same server-side renderer as the public blog page togglePreview(): void { if (this.previewMode()) { this.previewMode.set(false); return; } const blog = this.editingBlog(); - // An answer that arrives after the dialog moved on to another post belongs to nothing on screen const stillOpen = () => this.editingBlog() === blog; this.previewLoading.set(true); diff --git a/src/frontend/src/app/features/admin/contributors/admin-contributors.html b/src/frontend/src/app/features/admin/contributors/admin-contributors.html index 31a9b62..e10bf8c 100644 --- a/src/frontend/src/app/features/admin/contributors/admin-contributors.html +++ b/src/frontend/src/app/features/admin/contributors/admin-contributors.html @@ -3,7 +3,6 @@
-

Contributors

@@ -12,7 +11,6 @@

Contributors

-
- @if (backfillStatus() === 'running') {
@@ -40,7 +37,6 @@

Contributors

} -
@@ -50,7 +46,6 @@

Contributor Lea {{ result()?.total ?? 0 }} total

- @if (loadingLeaderboard()) {
@for (i of skeletonRows; track i) { @@ -73,7 +68,6 @@

Contributor Lea

} @else { -
@for (c of data.data; track c.gitHubUsername) { { this.reloadTimer = null; this.backfillStatus.set('idle'); diff --git a/src/frontend/src/app/features/admin/dashboard/admin-dashboard.html b/src/frontend/src/app/features/admin/dashboard/admin-dashboard.html index 735a1d3..3943728 100644 --- a/src/frontend/src/app/features/admin/dashboard/admin-dashboard.html +++ b/src/frontend/src/app/features/admin/dashboard/admin-dashboard.html @@ -2,22 +2,18 @@
- @if (loading()) {
Loading dashboard...
} @else if (stats(); as data) {
-

Dashboard

Overview of pkgdiff usage and analytics.

-
-
Total Searches @@ -28,7 +24,6 @@

Dashboard

{{ data.totalSearches.toLocaleString() }}
-
Cached Packages @@ -39,7 +34,6 @@

Dashboard

{{ data.cachedPackages.toLocaleString() }}
-
Unique Packages @@ -51,9 +45,7 @@

Dashboard

-
-
@@ -74,7 +66,6 @@

Top Packages

-
@@ -100,7 +91,6 @@

Searches (Last

-
diff --git a/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.html b/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.html index bfaa08d..f425e60 100644 --- a/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.html +++ b/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.html @@ -2,7 +2,6 @@
-

Health Monitoring

@@ -34,7 +33,6 @@

Health MonitoringLoading health status...

} @else if (healthDetails(); as health) { -
@@ -54,9 +52,7 @@

Overall System

-
-
@@ -88,7 +84,6 @@

Database

-
@@ -120,7 +115,6 @@

NuGet API

- @let gitHubLimited = health.gitHub.isAvailable && isRateLimited(health.gitHub);
@@ -168,7 +162,6 @@

GitHub API

-
@@ -207,7 +200,6 @@

Background Jobs

-

Package security providers

Switched on and off in Settings, category Security. They do not count towards the overall status.

@@ -232,7 +224,6 @@

Package

-

diff --git a/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.ts b/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.ts index 6b4d87f..e181a7a 100644 --- a/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.ts +++ b/src/frontend/src/app/features/admin/health-monitoring/health-monitoring.component.ts @@ -48,7 +48,7 @@ interface HangfireHealth { const REFRESH_INTERVAL_MS = 30000; -// The backend's safety buffer: at or below it, background scrapes pause until the limit resets +// Mirrors the backend buffer: at or below it, background scrapes pause until the reset const GITHUB_RATE_LIMIT_BUFFER = 10; @Component({ diff --git a/src/frontend/src/app/features/admin/jobs/admin-jobs.html b/src/frontend/src/app/features/admin/jobs/admin-jobs.html index 1a30b8c..6e6c394 100644 --- a/src/frontend/src/app/features/admin/jobs/admin-jobs.html +++ b/src/frontend/src/app/features/admin/jobs/admin-jobs.html @@ -2,7 +2,6 @@

-

Jobs

@@ -27,7 +26,6 @@

Jobs

Loading jobs...
} @else if (data(); as jobs) { -
Enqueued
@@ -51,7 +49,6 @@

Jobs

- @if (jobs.queuedJobs.length > 0) {
@@ -124,7 +121,6 @@

Active Queue } -
@@ -183,7 +179,6 @@

Recurring Jobs<

-
@@ -206,11 +201,9 @@

Recent Job Hist @for (job of jobs.recentJobs; track job.jobId) { - - @if (job.childJobs.length) { Recent Job Hist {{ formatDuration(job.duration) }} - @if (job.childJobs.length && isExpanded(job.jobId)) { @for (child of job.childJobs; track child.jobId) { diff --git a/src/frontend/src/app/features/admin/jobs/admin-jobs.ts b/src/frontend/src/app/features/admin/jobs/admin-jobs.ts index 72cb709..5f92cb5 100644 --- a/src/frontend/src/app/features/admin/jobs/admin-jobs.ts +++ b/src/frontend/src/app/features/admin/jobs/admin-jobs.ts @@ -157,7 +157,7 @@ export class AdminJobsComponent { } openDashboard(): void { - // Opened synchronously inside the click handler so the popup blocker allows it after the async call. + // Opened synchronously in the click handler so the popup blocker allows it const tab = window.open('about:blank', '_blank'); this.openingDashboard.set(true); this.api.post<{ url: string }>('admin/jobs/dashboard-ticket', {}).subscribe({ diff --git a/src/frontend/src/app/features/admin/login/admin-login.html b/src/frontend/src/app/features/admin/login/admin-login.html index b478263..fd1ba85 100644 --- a/src/frontend/src/app/features/admin/login/admin-login.html +++ b/src/frontend/src/app/features/admin/login/admin-login.html @@ -2,12 +2,10 @@
-
-
@@ -15,14 +13,12 @@

Sign In

-
-
Sign In

(keydown.enter)="handleLogin()">
- +
{{ loading() ? 'Signing in...' : 'Sign In' }} @@ -39,7 +35,6 @@

Sign In

-
diff --git a/src/frontend/src/app/features/admin/login/admin-login.ts b/src/frontend/src/app/features/admin/login/admin-login.ts index ec2dd80..36824a9 100644 --- a/src/frontend/src/app/features/admin/login/admin-login.ts +++ b/src/frontend/src/app/features/admin/login/admin-login.ts @@ -31,7 +31,7 @@ export class AdminLoginComponent { loading = signal(false); constructor() { - // This page has no layout around it, so nothing else creates the service that applies the stored theme + // No layout wraps this page, so nothing else instantiates the theme service inject(ThemeService); } diff --git a/src/frontend/src/app/features/admin/packages/admin-packages.html b/src/frontend/src/app/features/admin/packages/admin-packages.html index f4bafd3..5ed4ef2 100644 --- a/src/frontend/src/app/features/admin/packages/admin-packages.html +++ b/src/frontend/src/app/features/admin/packages/admin-packages.html @@ -2,7 +2,6 @@
-

Packages

@@ -11,7 +10,6 @@

Packages

-
@@ -21,7 +19,6 @@

Packages

class="w-full h-10 pl-10 pr-4 bg-white dark:bg-[#1a1a1a] border border-gray-200 dark:border-gray-700 rounded-lg text-sm text-[#111418] dark:text-white placeholder:text-gray-400 focus:outline-none focus:ring-2 focus:ring-primary/40 focus:border-primary transition-all" />
-
@if (loading()) {
diff --git a/src/frontend/src/app/features/admin/registries/admin-registries.html b/src/frontend/src/app/features/admin/registries/admin-registries.html index 5a22984..9dd5984 100644 --- a/src/frontend/src/app/features/admin/registries/admin-registries.html +++ b/src/frontend/src/app/features/admin/registries/admin-registries.html @@ -2,7 +2,6 @@
-

Registries

@@ -78,7 +77,6 @@

} -
@if (loading()) {
diff --git a/src/frontend/src/app/features/admin/registries/admin-registries.ts b/src/frontend/src/app/features/admin/registries/admin-registries.ts index 4499e41..4f95c73 100644 --- a/src/frontend/src/app/features/admin/registries/admin-registries.ts +++ b/src/frontend/src/app/features/admin/registries/admin-registries.ts @@ -148,7 +148,7 @@ export class AdminRegistriesComponent { const moved = [...this.registries()]; moveItemInArray(moved, event.previousIndex, event.currentIndex); - // Shown in the new order right away; the server's list comes back if it refuses + // Optimistic: the server's list is restored if it refuses const list = moved.map((reg, index) => ({ ...reg, displayOrder: index })); this.registries.set(list); diff --git a/src/frontend/src/app/features/admin/roles/admin-roles.html b/src/frontend/src/app/features/admin/roles/admin-roles.html index 1054030..258df15 100644 --- a/src/frontend/src/app/features/admin/roles/admin-roles.html +++ b/src/frontend/src/app/features/admin/roles/admin-roles.html @@ -2,7 +2,6 @@
-

Roles

@@ -21,7 +20,6 @@

Roles

- @if (loading()) {
Loading roles...
diff --git a/src/frontend/src/app/features/admin/roles/admin-roles.ts b/src/frontend/src/app/features/admin/roles/admin-roles.ts index 8d457cd..bf5b6fe 100644 --- a/src/frontend/src/app/features/admin/roles/admin-roles.ts +++ b/src/frontend/src/app/features/admin/roles/admin-roles.ts @@ -59,7 +59,6 @@ export class AdminRolesComponent { showModal = signal(false); roleName = signal(''); - // The role whose permissions are open in the dialog selectedRole = signal(null); permissionLoading = signal(false); rolePermissions = signal(null); @@ -156,8 +155,7 @@ export class AdminRolesComponent { } private loadPermissions(role: RoleInfo): void { - // The dialog may have been closed, or opened for another role, before the answer arrives; - // showing it then would let one role's permissions be saved onto another + // A late answer for another role would let its permissions be saved onto this one const stillOpen = () => this.selectedRole() === role; this.permissionLoading.set(true); diff --git a/src/frontend/src/app/features/admin/scrape-logs/admin-scrape-logs.html b/src/frontend/src/app/features/admin/scrape-logs/admin-scrape-logs.html index 38ff95d..9a7f250 100644 --- a/src/frontend/src/app/features/admin/scrape-logs/admin-scrape-logs.html +++ b/src/frontend/src/app/features/admin/scrape-logs/admin-scrape-logs.html @@ -2,7 +2,6 @@
-

Scrape Logs

@@ -17,7 +16,6 @@

Scrape Logs

-

- @if (loading()) {
Loading...
} @else { - @if (activeTab() === 'summary' && summary(); as data) {
-
@@ -70,7 +65,6 @@

Scrape Logs

-

Top Failing Packages

@@ -128,10 +122,8 @@

Top Failing Pac

} - @if (activeTab() === 'logs') {
-
-

Search History

@@ -18,7 +17,6 @@

Search History

}
-
@if (loading()) {
diff --git a/src/frontend/src/app/features/admin/security/admin-security.html b/src/frontend/src/app/features/admin/security/admin-security.html index 751a5d2..0d8b915 100644 --- a/src/frontend/src/app/features/admin/security/admin-security.html +++ b/src/frontend/src/app/features/admin/security/admin-security.html @@ -2,7 +2,6 @@
-

Security & IP Blocking

@@ -17,7 +16,6 @@

Security & IP Block

-
- @if (loading()) {
Loading...
} @else { - @if (activeTab() === 'blocked' && blockedIps(); as data) {
-
Security & IP Block
} - @if (activeTab() === 'logs' && requestLogs(); as data) {
-
Security & IP Block
} - @if (activeTab() === 'stats') {
-