diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fdcd4b8a15eb..be4a439018f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,17 +58,42 @@ jobs: echo "Base Ref: $BASE_REF" echo "PR DESCRIPTION: $PR_DESCRIPTION" if [ "$EVENT_NAME" = "pull_request" ]; then - if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \ - -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \ - "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then - echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description" + on_base=0 + if [ "$(git rev-parse --abbrev-ref HEAD)" = "$BASE_REF" ]; then + on_base=1 + fi + found=0 + if ! commits=$(git rev-list "origin/${BASE_REF}..HEAD"); then + echo "::error::Cannot list commits origin/${BASE_REF}..HEAD" exit 1 - else - if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then - echo "Changelog found in PR description" - else - echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\"" + fi + # Same shape devtools/changelog.py keeps: class, colon, then space. + # A class with no colon or no space does not count. Changelog-None + # is the opt-out, in a commit or in the PR body. + while IFS= read -r sha; do + [ -n "$sha" ] || continue + msg=$(git show -s --format=%B "$sha") + shopt -s nocasematch + while IFS= read -r line; do + if [[ "$line" =~ ^[[:space:]]*Changelog-(Added|Changed|Deprecated|Fixed|Removed|Experimental):[[:space:]] ]] || [[ "$line" =~ ^[[:space:]]*Changelog-None[[:space:]]*$ ]] || [[ "$line" =~ ^[[:space:]]*Changelog-None: ]]; then + found=1 + break + fi + done <<< "$msg" + shopt -u nocasematch + if [ "$found" -eq 1 ]; then + break fi + done <<< "$commits" + if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then + echo "Changelog found in PR description" + elif [ "$found" -eq 1 ]; then + echo "Changelog found in commit range origin/${BASE_REF}..HEAD" + elif [ "$on_base" -eq 1 ]; then + echo "HEAD is ${BASE_REF}; changelog check skipped" + else + echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description" + exit 1 fi else echo "Not a PR event." diff --git a/.github/workflows/per-commit.yaml b/.github/workflows/per-commit.yaml new file mode 100644 index 000000000000..b3c6e627174e --- /dev/null +++ b/.github/workflows/per-commit.yaml @@ -0,0 +1,125 @@ +--- +name: Per-commit checks +# Builds and tests every commit of a PR except the last, which the main CI +# workflow already covers. +# +# Off by default. A maintainer turns it on for all PRs by setting the +# repository variable PER_COMMIT_CHECKS to "true", for one PR by adding the +# "per-commit-checks" label, or runs it by hand on a branch. +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + base: + description: Branch the commits are compared against + default: master +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Reduce size, as in the main CI workflow + RUST_PROFILE: small + SLOW_MACHINE: 1 + SCCACHE_GHA_ENABLED: "true" + BASE_REF: ${{ github.base_ref || inputs.base }} + PASSED_KEY: check-commits-${{ github.event.pull_request.number || github.ref_name }} + +jobs: + per-commit: + name: Build and test each commit + runs-on: ubuntu-24.04 + timeout-minutes: 120 + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.action == 'labeled' && + github.event.label.name == 'per-commit-checks') || + (github.event.action != 'labeled' && + (vars.PER_COMMIT_CHECKS == 'true' || + contains(github.event.pull_request.labels.*.name, 'per-commit-checks'))) + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + + - name: Set up Python 3.10 + uses: actions/setup-python@v7 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v10.3.2 + + - uses: actions/cache@v6 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + # check-source checks BOLT quotes + git clone https://github.com/lightning/bolts.git ../bolts + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.11 + + # Commits that passed on an earlier push of this PR are not rechecked. + - name: Restore list of commits that passed + uses: actions/cache/restore@v6 + with: + path: ~/check-commits + key: ${{ env.PASSED_KEY }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ env.PASSED_KEY }}- + + # The Rust build is the slow part, and the main CI workflow builds the + # last commit with it anyway. + - name: Configure + run: | + git fetch origin "$BASE_REF" + rust=--disable-rust + if ! git diff --quiet "origin/$BASE_REF...HEAD" -- '*.rs' '*Cargo.toml' Cargo.lock cln-grpc/ cln-rpc/ contrib/msggen/ doc/schemas/; then + rust=--enable-rust + fi + ./configure --enable-debugbuild --disable-valgrind "$rust" CC="sccache gcc" + + - name: Check each commit + env: + COMPAT: 1 + RUSTC_WRAPPER: sccache + PYTEST_OPTS: --timeout=1800 + CHECK_COMMITS_SKIP_TIP: 1 + CHECK_COMMITS_LOGDIR: ${{ runner.temp }}/check-commits-logs + run: | + mkdir -p ~/check-commits + touch ~/check-commits/passed + export CHECK_COMMITS_CACHE=~/check-commits/passed + export GRPC_BREAKING_BASE_REF="origin/$BASE_REF" + # Use the base branch's script, so a PR branched before a change + # to it is checked the same way as every other PR. + script="$RUNNER_TEMP/check-commits.sh" + git show "origin/$BASE_REF:tools/check-commits.sh" > "$script" 2>/dev/null || cp tools/check-commits.sh "$script" + chmod +x "$script" + sg wireshark "uv run eatmydata $script origin/$BASE_REF" + + - name: Save list of commits that passed + if: always() + uses: actions/cache/save@v6 + with: + path: ~/check-commits + key: ${{ env.PASSED_KEY }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Upload step logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: check-commits-logs + path: ${{ runner.temp }}/check-commits-logs + if-no-files-found: ignore diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65e0f58bec8d..17d1ae099d77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,12 +65,6 @@ repos: exclude: ccan|contrib|tests/fuzz/corpora stages: [ manual ] -- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v9.23.0 - hooks: - - id: commitlint - stages: [ commit-msg ] - - repo: local hooks: # Reimplementation of `make check-amount-access` for pygrep. diff --git a/Makefile b/Makefile index 96495adc1a1f..b796364119c5 100644 --- a/Makefile +++ b/Makefile @@ -710,11 +710,41 @@ check-grpc-breaking: echo "Checking gRPC protos against $$tag"; \ $(BUF) breaking cln-grpc/proto --against ".git#tag=$$tag,subdir=cln-grpc/proto" +# Only lines added since BASE_REF are checked (doc/Makefile sets the +# default): see tools/added-lines.sh. +check-xfail-strict: + @tools/check-xfail-strict.sh $(BASE_REF) + +check-switch-default: + @tools/check-switch-default.py $(BASE_REF) + +check-comments: + @tools/check-comments.py $(BASE_REF) + +check-unused-functions: + @tools/check-unused-functions.py $(BASE_REF) + +check-take-parent: + @tools/check-take-parent.py $(BASE_REF) + +# The message of each commit since BASE_REF. +check-commit-msgs: + @tools/check-commit-msgs.py $(BASE_REF) + # This should NOT compile things! -check-source: check-makefile check-whitespace check-spelling check-python-flake8 check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf check-wire-format check-source-bolt check-security check-msggen-grpc check-grpc-breaking +check-source: check-makefile check-whitespace check-spelling check-python-flake8 check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf check-wire-format check-source-bolt check-security check-msggen-grpc check-grpc-breaking check-xfail-strict check-switch-default check-comments check-unused-functions check-take-parent check-commit-msgs full-check: check check-source +# Build, check-source, unit tests and touched integration tests on each +# commit since BASE_REF: see tools/check-commits.sh. +check-commits: + @uv run tools/check-commits.sh $(BASE_REF) + +# The same, with the whole integration suite on every commit. +check-commits-full: + @CHECK_COMMITS_PYTEST=all uv run tools/check-commits.sh $(BASE_REF) + # Simple target to be used on CI systems to check that all the derived # files were checked in and updated. It depends on the generated # targets, and checks if any of the tracked files changed. If they did diff --git a/commitlint.config.js b/commitlint.config.js deleted file mode 100644 index f156abd38cc5..000000000000 --- a/commitlint.config.js +++ /dev/null @@ -1,56 +0,0 @@ -module.exports = { - - plugins: [ - { - rules: { - 'core-lightning': ({ type }) => { - // Allow standard Core Lightning types - const standardTypes = [ - // Daemons - 'channeld', 'closingd', 'connectd', 'gossipd', 'hsmd', 'lightningd', 'onchaind', - 'openingd', - // Related - 'bitcoin', 'cli', 'cln-grpc', 'cln-rpc', 'db', 'wallet', 'wire', - // Others - 'ci', 'common', 'contrib', 'devtools', 'docs', 'docker', 'github', 'global', - 'meta', 'nit', 'nix', 'release', 'script', 'tests', - ]; - - // Extensions - const extensions = ['plugin-', 'pyln-', 'tool-'] - if (type) { - for (const prefix of extensions) { - if (type.startsWith(prefix)) { - return [true]; - } - } - } - - // Otherwise, must be a standard type - if (standardTypes.includes(type)) { - return [true]; - } - - return [ - false, - `Type must be one of [${standardTypes.join(', ')}] or match patterns [${extensions.join(', ')}]` - ]; - }, - }, - }, - ], - - rules: { - // Disable the default type-enum rule since we're using custom validation - 'type-enum': [0], - - // Enable our custom rule - 'core-lightning': [2, 'always'], - - // Keep other standard rules - 'type-case': [2, 'always', 'lower-case'], - 'type-empty': [2, 'never'], - 'subject-empty': [2, 'never'], - 'subject-case': [2, 'never', ['upper-case']], - }, -}; diff --git a/tools/added-lines.sh b/tools/added-lines.sh new file mode 100755 index 000000000000..b06bbeb3ae51 --- /dev/null +++ b/tools/added-lines.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Print "path:line" for every line the working tree adds since the merge +# base of BASE and HEAD, in files matching the pathspecs. A check built on +# this complains about new code only, not about what is already there. +# +# Usage: tools/added-lines.sh BASE [pathspec...] + +set -o pipefail + +base=${1:?usage: added-lines.sh BASE [pathspec...]} +shift + +if ! mergebase=$(git merge-base "$base" HEAD 2> /dev/null); then + echo "added-lines: no merge base with '$base', so no lines are checked" >&2 + exit 0 +fi + +# With -U0 a hunk is only removed and added lines; the +++ header is told +# apart from an added line by coming before the first hunk of its file. +git -c core.quotePath=false diff -U0 --no-color --no-ext-diff --no-prefix "$mergebase" -- "$@" | awk ' +/^diff --git / { header = 1; next } +header && /^\+\+\+ / { file = substr($0, 5); next } +/^@@ / { + header = 0 + split($3, range, ",") + line = substr(range[1], 2) + 0 + next +} +!header && /^\+/ { print file ":" line; line++ } +' || exit 1 + +# A file git does not know yet is new in every line. +git ls-files --others --exclude-standard -- "$@" | while IFS= read -r file; do + awk -v file="$file" '{ print file ":" NR }' "$file" +done diff --git a/tools/check-comments.py b/tools/check-comments.py new file mode 100755 index 000000000000..df791ecb919b --- /dev/null +++ b/tools/check-comments.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Flag added comments which are long, or which will go stale. + +A comment describes the code beside it, to a reader who has never seen +any other version of the file. What a change fixed, what the code used to +do, and which pull request or review asked for it belong in the commit +message: in a comment they are out of date as soon as the commit lands. +And a comment which needs many lines usually explains code which should be +simpler. + +Only added lines are checked (see tools/added-lines.sh). BOLT quotes are +left alone, and the /*~ walkthrough comments may be any length. +""" +import io +import os +import re +import subprocess +import sys +import tokenize + +PATHSPECS = ['*.c', '*.h', '*.py', ':(exclude)ccan/', + ':(exclude)*gen.c', ':(exclude)*gen.h', + ':(exclude)*_pb2*.py', ':(exclude)*grpc2py.py'] + +# A comment at the left margin documents a function or a type; an indented +# one is in the middle of the code it explains. +MAX_LINES = 8 +MAX_INDENTED_LINES = 4 + +STALE = [ + (re.compile(r'\b(the|this) fix\b|\bthis (commit|pr|patch|change)\b' + r'|\b(previous|next|earlier|later|follow-?up) (commit|pr|patch)\b', re.I), + 'describes the change, not the code: put that in the commit message'), + (re.compile(r'\bused to be\b|\b(we|it|they|there|this|that|which|\w+\(\)) used to\b' + r'|\b(old|previous|former|original|earlier) (code|implementation|behaviou?r|logic)\b' + r'|\brefactor(ed|ing)?\b|\brenamed (from|to)\b', re.I), + 'describes code which is gone: say what this code does, or what breaks without it'), + (re.compile(r'github\.com/ElementsProject/lightning/(pull|issues)/\d+' + r'|\b(pr|mr|pull request|issue)\s*[#!]?\d+\b' + r'|(? limit and not whole.startswith('/*~'): + where = 'inside code' if indented else 'at the left margin' + yield new[0], (f'comment is {len(lines)} lines long (limit {limit} {where}):' + ' keep it short, and let simpler code say the rest') + for line, content in enumerate(lines, first): + if line not in added: + continue + for pattern, complaint in STALE: + if pattern.search(content): + yield line, f'comment {complaint}' + break + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else 'master' + helper = os.path.join(os.path.dirname(sys.argv[0]), 'added-lines.sh') + lines = subprocess.run([helper, base] + PATHSPECS, + check=True, stdout=subprocess.PIPE, text=True) + added = {} + for entry in lines.stdout.splitlines(): + path, line = entry.rsplit(':', 1) + added.setdefault(path, set()).add(int(line)) + + failed = False + for path in sorted(added): + with open(path, errors='replace') as f: + text = f.read() + for line, complaint in sorted(problems(path, text, added[path])): + print(f'{path}:{line}: {complaint}') + failed = True + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-commit-msgs.py b/tools/check-commit-msgs.py new file mode 100755 index 000000000000..44be94848562 --- /dev/null +++ b/tools/check-commit-msgs.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Flag commit messages in BASE..HEAD which are hard to read, or go stale. + +Once a commit lands, its message is all that explains it. The pull request +and its review are not part of the history, and the commit before it is +not the same one after a rebase or a cherry-pick. So the message has to +stand on its own, as plain wrapped text in paragraphs short enough to +follow. Its subject names the part of the tree it changes, as in +"lightningd: fix a crash on restart", which is what a reader scans for in +the log. + +Indented lines are quoted output, not prose, so logs and backtraces can be +any width and length. Links and trailers are left alone too. +""" +import re +import subprocess +import sys + +MAX_COLUMNS = 80 +MAX_PARAGRAPH = 10 + +TRAILER = re.compile(r'(Changelog-[A-Za-z]+|Signed-off-by|Co-authored-by|Fixes|Closes' + r'|Reported-by|Suggested-by|Reviewed-by|Tested-by|Acked-by' + r'|Assisted-by|Cc|Link)\s*:|Changelog-None|\(cherry picked from', re.I) +HEADER = re.compile(r'#{1,6} \S') +# "subsystem: intent"; a release commit and git's own subjects are left alone. +SUBJECT = re.compile(r'((fixup|squash|amend)! )*(\S+ ){0,2}\S+: \S|v\d+\.\d+\S*$|Revert "') +UNSEEN = re.compile(r'\bthis (pr|mr|pull request)\b|\breview(er)? (comments?|feedback)\b' + r'|\bper review\b|\bas (requested|discussed)\b', re.I) +BY_POSITION = re.compile(r'\b(previous|last|earlier|prior|preceding|parent) (commit|patch)(es|s)?\b', re.I) + + +def is_prose(line): + return bool(line.strip() + and not line.startswith((' ', '\t', '>')) + and not TRAILER.match(line)) + + +def problems(message): + lines = message.rstrip('\n').split('\n') + if not SUBJECT.match(lines[0]): + yield ('needs a subject of the form "subsystem: intent",' + ' naming the daemon or directory it changes') + if len(lines) > 1 and lines[1].strip(): + yield 'needs a blank line after the subject' + + paragraph = longest = 0 + wide = header = unseen = by_position = False + for line in lines[1:]: + if not is_prose(line): + paragraph = 0 + continue + paragraph += 1 + longest = max(longest, paragraph) + if len(line) > MAX_COLUMNS and '://' not in line and ' ' in line.strip(): + wide = True + header = header or bool(HEADER.match(line)) + unseen = unseen or bool(UNSEEN.search(line)) + by_position = by_position or bool(BY_POSITION.search(line)) + + if wide: + yield (f'has text wider than {MAX_COLUMNS} columns:' + ' wrap what you wrote, and indent any output you quote') + if longest > MAX_PARAGRAPH: + yield (f'has a paragraph of {longest} lines (limit {MAX_PARAGRAPH}):' + ' break it up, and indent any output you quote') + if header: + yield 'has markdown headers: commit messages are plain text' + if unseen: + yield 'refers to a pull request or review which the reader cannot see' + if by_position: + yield ('refers to an earlier commit by its position:' + ' say what the code does now, so this commit stands on its own') + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else 'master' + log = subprocess.run(['git', 'log', '--reverse', '--no-merges', + '--format=%x01%h %s%x00%B', f'{base}..HEAD'], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + if log.returncode != 0: + print(f"check-commit-msgs: no commits found from '{base}', so none are checked", + file=sys.stderr) + sys.exit(0) + + failed = False + for commit in log.stdout.split('\x01')[1:]: + name, message = commit.split('\x00') + for problem in problems(message): + print(f'{name}: message {problem}') + failed = True + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-commits.sh b/tools/check-commits.sh new file mode 100755 index 000000000000..05fb5bb8700f --- /dev/null +++ b/tools/check-commits.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Check each commit in BASE..HEAD on its own, oldest first: build, +# check-source, unit tests, then the integration test files that commit +# touches. Run it through make, so the tools come from the uv environment: +# +# make check-commits [BASE_REF=origin/master] +# make check-commits-full # whole integration suite on every commit +# +# The branch is not rewritten. Each commit is checked out detached and the +# original HEAD is restored afterwards, so tracked files must be clean. +# +# A commit that passes is recorded by tree, parent tree and configuration, +# so a rerun after fixing a later commit skips the earlier ones. +# +# Environment: +# CHECK_COMMITS_PYTEST touched (default), all, or none +# CHECK_COMMITS_SKIP_TIP 1 to leave out the last commit +# CHECK_COMMITS_CACHE file listing passed commits (default: in .git; +# /dev/null to disable) +# CHECK_COMMITS_LOGDIR where step logs go (default: a new temp dir) +# CHECK_COMMITS_JOBS parallel make jobs (default: number of CPUs) +# PYTEST_PAR pytest workers (default: CHECK_COMMITS_JOBS) + +set -u + +orig= + +die() { + echo "check-commits: $*" >&2 + exit 1 +} + +tracked_changes() { + git status --porcelain --untracked-files=no --ignore-submodules=all +} + +restore_head() { + if [ -n "$orig" ] && ! git checkout -q "$orig"; then + echo "check-commits: could not return to $orig, HEAD is still detached" >&2 + fi +} + +# Tracked files that a step rewrites are stale in the commit, and would +# block the checkout of the next one. +require_clean() { + if [ -n "$(tracked_changes)" ]; then + echo "check-commits: $1 changed tracked files, so this commit has them stale:" >&2 + tracked_changes >&2 + return 1 + fi +} + +# Run one step with its output in a log, and show the tail if it fails. +step() { + local name=$1 log + shift + log="$logdir/$label-$name.log" + printf ' %-12s ' "$name" + if "$@" > "$log" 2>&1; then + echo ok + return 0 + fi + echo FAILED + tail -n 60 "$log" + echo "check-commits: $name failed, full log in $log" >&2 + return 1 +} + +check_commit() { + local sha=$1 tests + + step build make -j "$jobs" || return 1 + require_clean "the build" || return 1 + step check-source make -j "$jobs" check-source BASE_REF="$base" || return 1 + step check-units make -j "$jobs" check-units || return 1 + + case "$pytest_mode" in + none) + ;; + all) + step pytest make pytest || return 1 + ;; + touched) + tests=$(git diff-tree --no-commit-id --name-only -r --diff-filter=d "$sha" -- 'tests/test_*.py' | tr '\n' ' ') + if [ -n "$tests" ]; then + step pytest make pytest PYTEST_TESTS="$tests" || return 1 + fi + ;; + esac + require_clean "the checks" +} + +# Everything runs from main, so bash has read the whole file before the +# first checkout: an older commit may not have this script. +main() { + local base commits total config key sha n=0 skipped=0 + + cd "$(git rev-parse --show-toplevel)" || exit 1 + + base=${1:-origin/master} + pytest_mode=${CHECK_COMMITS_PYTEST:-touched} + case "$pytest_mode" in + touched|all|none) + ;; + *) + die "CHECK_COMMITS_PYTEST must be touched, all or none" + ;; + esac + jobs=${CHECK_COMMITS_JOBS:-$(getconf _NPROCESSORS_ONLN)} + export PYTEST_PAR=${PYTEST_PAR:-$jobs} + cache=${CHECK_COMMITS_CACHE:-$(git rev-parse --git-path check-commits-passed)} + # The sub-makes take their options from here, not from a make that ran us. + unset MAKEFLAGS MFLAGS MAKELEVEL + + git rev-parse -q --verify "$base^{commit}" > /dev/null || die "no such base: $base" + [ -z "$(tracked_changes)" ] || die "tracked files have changes, commit or stash them first" + + commits=$(git rev-list --reverse "$base..HEAD") || exit 1 + if [ "${CHECK_COMMITS_SKIP_TIP:-0}" = 1 ]; then + commits=$(printf '%s\n' "$commits" | sed '$d') + fi + if [ -z "$commits" ]; then + echo "check-commits: no commits to check in $base..HEAD" + exit 0 + fi + total=$(printf '%s\n' "$commits" | wc -l | tr -d ' ') + + logdir=${CHECK_COMMITS_LOGDIR:-$(mktemp -d "${TMPDIR:-/tmp}/check-commits.XXXXXX")} + mkdir -p "$logdir" || exit 1 + + orig=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) + # A check may need to see what the rest of the branch does with a commit. + CHECK_COMMITS_TIP=$(git rev-parse HEAD) || exit 1 + export CHECK_COMMITS_TIP + trap restore_head EXIT + trap 'exit 130' INT TERM + + for sha in $commits; do + n=$((n + 1)) + label=$(printf '%02d-%s' "$n" "$(git rev-parse --short "$sha")") + echo "[$n/$total] $(git log -1 --format='%h %s' "$sha")" + + config=$(git hash-object config.vars 2> /dev/null || echo unconfigured) + key="$(git rev-parse "$sha^{tree}") $(git rev-parse -q --verify "$sha^^{tree}" || echo root) $pytest_mode $config" + if grep -qxF "$key" "$cache" 2> /dev/null; then + echo " already passed" + skipped=$((skipped + 1)) + continue + fi + + git checkout -q --detach "$sha" || exit 1 + if ! check_commit "$sha"; then + echo "check-commits: FAILED at $(git log -1 --format='%h %s' "$sha")" >&2 + exit 1 + fi + echo "$key" >> "$cache" + done + + echo "check-commits: all $total commits pass ($skipped already passed in an earlier run)" +} + +main "$@" diff --git a/tools/check-switch-default.py b/tools/check-switch-default.py new file mode 100755 index 000000000000..a649990424ba --- /dev/null +++ b/tools/check-switch-default.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Flag a `default:` label added to a switch over an enum. + +Without a default, the compiler warns about every switch which does not +handle a newly added enumerator. + +Only added lines are checked (see tools/added-lines.sh), and only switches +where some case label is an enumerator: one declared in a tracked source +file, or a WIRE_ name, since the wire headers are generated. A switch over +errno values or address families may keep its default. +""" +import os +import re +import subprocess +import sys + +PATHSPECS = ['*.c', '*.h', ':(exclude)ccan/', ':(exclude)contrib/'] + +# Comments, strings and character constants: none of them hold code. +NOT_CODE = re.compile(r'''//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\\n])*"|'(?:\\.|[^'\\\n])*' ''', re.S | re.X) +ENUM = re.compile(r'\benum\b[^;{}()]*\{([^{}]*)\}') +ENUMERATOR = re.compile(r'\s*([A-Za-z_]\w*)') +SWITCH = re.compile(r'\bswitch\s*\(') +CASE = re.compile(r'\bcase\s+([A-Za-z_]\w*)\s*:') +DEFAULT = re.compile(r'\bdefault\s*:') + + +def code_only(text): + """Blank out whatever is not code, keeping every line where it was.""" + return NOT_CODE.sub(lambda m: re.sub(r'[^\n]', ' ', m.group()), text) + + +def read(path): + with open(path, errors='replace') as f: + return f.read() + + +def enumerators(): + """Every enumerator declared in a tracked C file.""" + names = set() + tracked = subprocess.run(['git', 'ls-files', '-z', '--', '*.c', '*.h'], + check=True, stdout=subprocess.PIPE, text=True) + for path in tracked.stdout.split('\0'): + if not os.path.isfile(path): + continue + text = read(path) + if 'enum' not in text: + continue + for body in ENUM.findall(code_only(text)): + body = re.sub(r'^\s*#.*$', '', body, flags=re.M) + for item in body.split(','): + name = ENUMERATOR.match(item) + if name: + names.add(name.group(1)) + return names + + +def closing(code, start, opener, closer): + """Offset just past the bracket which closes the one at start.""" + depth = 0 + for i in range(start, len(code)): + if code[i] == opener: + depth += 1 + elif code[i] == closer: + depth -= 1 + if depth == 0: + return i + 1 + return None + + +def switch_bodies(code): + """(start, end) offsets of the braces of each switch statement.""" + bodies = [] + for switch in SWITCH.finditer(code): + after = closing(code, switch.end() - 1, '(', ')') + if after is None: + continue + brace = code.find('{', after) + if brace == -1 or code[after:brace].strip(): + continue + end = closing(code, brace, '{', '}') + if end is not None: + bodies.append((brace, end)) + return bodies + + +def innermost(bodies, offset): + inside = [b for b in bodies if b[0] < offset < b[1]] + return max(inside, default=None) + + +def enum_defaults(code, names): + """(line, enumerator) for each default: in a switch over an enum.""" + bodies = switch_bodies(code) + for default in DEFAULT.finditer(code): + body = innermost(bodies, default.start()) + if body is None: + continue + for case in CASE.finditer(code, body[0], body[1]): + label = case.group(1) + if innermost(bodies, case.start()) != body: + continue + if label in names or label.startswith('WIRE_'): + yield code.count('\n', 0, default.start()) + 1, label + break + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else 'master' + helper = os.path.join(os.path.dirname(sys.argv[0]), 'added-lines.sh') + lines = subprocess.run([helper, base] + PATHSPECS, + check=True, stdout=subprocess.PIPE, text=True) + added = {} + for entry in lines.stdout.splitlines(): + path, line = entry.rsplit(':', 1) + added.setdefault(path, set()).add(int(line)) + + names = None + failed = False + for path in sorted(added): + code = code_only(read(path)) + if not DEFAULT.search(code): + continue + if names is None: + names = enumerators() + for line, label in enum_defaults(code, names): + if line in added[path]: + print(f'{path}:{line}: default: in a switch over an enum ({label}, ...):' + ' list every case, so the compiler warns when one is added') + failed = True + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-take-parent.py b/tools/check-take-parent.py new file mode 100755 index 000000000000..7698ecad4161 --- /dev/null +++ b/tools/check-take-parent.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Flag take() of something which was allocated off a parent. + +take() gives its argument to the function being called. Something +allocated only to be given away should be allocated off NULL, so it is +clear that nothing else owns it: + + subd_send_msg(sd, take(towire_foo(NULL, ...))); + +Only added lines are checked (see tools/added-lines.sh). The allocations +recognised are the ones which name their parent first: anything given +tmpctx, and the towire_ and tal functions. +""" +import os +import re +import subprocess +import sys + +PATHSPECS = ['*.c', '*.h', ':(exclude)ccan/', ':(exclude)contrib/', ':(exclude)external/'] + +# Comments, strings and character constants: none of them hold code. +NOT_CODE = re.compile(r'''//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\\n])*"|'(?:\\.|[^'\\\n])*' ''', re.S | re.X) +TAKE = re.compile(r'\btake\(\s*(?:\w+\(\s*(tmpctx)\b|(?:towire_\w+|tal\w*)\(\s*(?!NULL\b)(\w+))') + + +def code_only(text): + """Blank out whatever is not code, keeping every line where it was.""" + return NOT_CODE.sub(lambda m: re.sub(r'[^\n]', ' ', m.group()), text) + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else 'master' + helper = os.path.join(os.path.dirname(sys.argv[0]), 'added-lines.sh') + lines = subprocess.run([helper, base] + PATHSPECS, + check=True, stdout=subprocess.PIPE, text=True) + added = {} + for entry in lines.stdout.splitlines(): + path, line = entry.rsplit(':', 1) + added.setdefault(path, set()).add(int(line)) + + failed = False + for path in sorted(added): + with open(path, errors='replace') as f: + code = code_only(f.read()) + for take in TAKE.finditer(code): + line = code.count('\n', 0, take.start()) + 1 + if line in added[path]: + parent = take.group(1) or take.group(2) + print(f'{path}:{line}: take() of something allocated off {parent}:' + ' allocate it off NULL, since take() gives it away') + failed = True + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-unused-functions.py b/tools/check-unused-functions.py new file mode 100755 index 000000000000..cec53a82778a --- /dev/null +++ b/tools/check-unused-functions.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Flag an added function which nothing calls. + +Unused code is buggy code: nothing runs it, so nothing shows that it +works. The compiler says so for a static function; this is the same check +for one which is not static. + +Only functions defined on added lines are checked (see +tools/added-lines.sh). A caller in a unit test counts. towire_ and +fromwire_ functions are left alone: generated code calls them, and it is +not tracked. + +A commit in the middle of a branch may add a function for a later commit +to call. tools/check-commits.sh sets CHECK_COMMITS_TIP to the last commit +of the branch, and callers are then looked for there. +""" +import os +import re +import subprocess +import sys + +PATHSPECS = ['*.c', ':(exclude)ccan/', ':(exclude)contrib/', ':(exclude)external/', + ':(exclude)tests/', ':(exclude)*/test/*'] + +# Comments, strings and character constants: none of them hold code. +NOT_CODE = re.compile(r'''//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\\n])*"|'(?:\\.|[^'\\\n])*' ''', re.S | re.X) +# A return type and a name at the left margin: a definition or a prototype. +DECLARATION = re.compile(r'^(?!static\b)(?:\w+[ \t*]+)+(\w+)\(', re.M) +BODY_OR_END = re.compile(r'[{;]') +GENERATED_CALLERS = ('towire_', 'fromwire_') + + +def code_only(text): + """Blank out whatever is not code, keeping every line where it was.""" + return NOT_CODE.sub(lambda m: re.sub(r'[^\n]', ' ', m.group()), text) + + +def definitions(code): + """(line, name) of each function which is defined and not static.""" + for declaration in DECLARATION.finditer(code): + body = BODY_OR_END.search(code, declaration.end()) + if body and body.group() == '{': + yield code.count('\n', 0, declaration.start()) + 1, declaration.group(1) + + +def is_use(line, name): + """Does this line of code name the function, other than to declare it?""" + if not re.search(rf'\b{name}\b', line): + return False + return not re.match(rf'[\w*][\w\s*]*\b{name}\(', line) + + +def used(names): + """Those of the names which some tracked C file uses.""" + tip = os.environ.get('CHECK_COMMITS_TIP') + command = ['git', 'grep', '-h', '-w', '-I'] + for name in names: + command += ['-e', name] + # Files which git does not know yet are part of the working tree too. + grep = subprocess.run(command + ([tip] if tip else ['--untracked']) + ['--', '*.c', '*.h'], + stdout=subprocess.PIPE, text=True, errors='replace') + found = set() + for line in grep.stdout.splitlines(): + # A block comment which this line only starts, or is in the middle of. + line = re.sub(r'/\*.*', '', code_only(line)) + if line.lstrip().startswith('*'): + continue + found.update(name for name in names if is_use(line, name)) + return found + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else 'master' + helper = os.path.join(os.path.dirname(sys.argv[0]), 'added-lines.sh') + lines = subprocess.run([helper, base] + PATHSPECS, + check=True, stdout=subprocess.PIPE, text=True) + added = {} + for entry in lines.stdout.splitlines(): + path, line = entry.rsplit(':', 1) + added.setdefault(path, set()).add(int(line)) + + new = {} + for path in sorted(added): + with open(path, errors='replace') as f: + code = code_only(f.read()) + for line, name in definitions(code): + if (line in added[path] and name != 'main' + and not name.startswith(GENERATED_CALLERS)): + new[name] = (path, line) + if not new: + sys.exit(0) + + unused = sorted(new[name] + (name,) for name in set(new) - used(set(new))) + for path, line, name in unused: + print(f'{path}:{line}: {name}() has no caller:' + ' unused code is buggy code, so leave it out until something needs it') + sys.exit(1 if unused else 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-xfail-strict.sh b/tools/check-xfail-strict.sh new file mode 100755 index 000000000000..62710ccc83df --- /dev/null +++ b/tools/check-xfail-strict.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Flag a pytest.mark.xfail decorator added under tests/ that does not +# contain strict=True. A decorator split across lines is read until its +# parentheses balance, or until the following def. +# +# Only added lines are checked: see tools/added-lines.sh. +# +# Usage: tools/check-xfail-strict.sh [BASE] + +added=$("$(dirname "$0")/added-lines.sh" "${1:-master}" 'tests/*.py') || exit 1 +if [ -z "$added" ]; then + exit 0 +fi + +hits=$(printf '%s\n' "$added" | cut -d: -f1 | sort -u | while IFS= read -r f; do + lines=$(printf '%s\n' "$added" | awk -F: -v f="$f" '$1 == f { print $2 }' | tr '\n' ' ') + awk -v lines="$lines" ' +function delta(s, i, n, c, d, q, sq) { + d = 0 + q = "" + sq = sprintf("%c", 39) + n = length(s) + for (i = 1; i <= n; i++) { + c = substr(s, i, 1) + if (q != "") { + if (c == q) + q = "" + continue + } + if (c == "\"" || c == sq) { + q = c + continue + } + if (c == "(") + d++ + else if (c == ")") + d-- + } + return d +} + +function depth_after_mark(s, i, rest) { + i = index(s, "pytest.mark.xfail") + if (i == 0) + return 0 + rest = substr(s, i + length("pytest.mark.xfail")) + sub(/^[[:space:]]*/, "", rest) + if (substr(rest, 1, 1) != "(") + return 0 + return delta(rest) +} + +BEGIN { + capturing = 0 + n = split(lines, list, " ") + for (i = 1; i <= n; i++) + added[list[i]] = 1 +} +{ + if (!capturing) { + if (index($0, "pytest.mark.xfail") == 0 || !(NR in added)) + next + block = $0 + start = NR + depth = depth_after_mark($0) + if (depth <= 0) { + if (index(block, "strict=True") == 0) + printf "%s:%d:pytest.mark.xfail without strict=True\n", FILENAME, start + next + } + capturing = 1 + next + } + block = block "\n" $0 + if ($0 ~ /^[[:space:]]*(async[[:space:]]+)?def[[:space:]]/) { + if (index(block, "strict=True") == 0) + printf "%s:%d:pytest.mark.xfail without strict=True\n", FILENAME, start + capturing = 0 + next + } + depth += delta($0) + if (depth <= 0) { + if (index(block, "strict=True") == 0) + printf "%s:%d:pytest.mark.xfail without strict=True\n", FILENAME, start + capturing = 0 + } +} +END { + if (capturing && index(block, "strict=True") == 0) + printf "%s:%d:pytest.mark.xfail without strict=True\n", FILENAME, start +} +' "$f" +done) + +if [ -n "$hits" ]; then + printf '%s\n' "$hits" + exit 1 +fi +exit 0