From c6501a5104d347f78e95955aa4b2667ebf6079f5 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Sun, 1 Nov 2026 00:22:00 +0900 Subject: [PATCH] ci: run TechEngine's collectors by hand and verify in final-check collect.yml (workflow_dispatch only, on develop) calls TechEngine's ingest, refresh and verify-promote workflows, pinned by SHA. Each task keeps one open PR into develop on a fixed collect/* branch, updated in place, from github-actions[bot]. Nothing is scheduled and nothing posts comments; the collector validates before it pushes and writes the result into the PR body. final-check also runs the engine's Tier 0-3 report on the records a PR changes and puts it in the one TechEngineBot comment, with mentions and cross-references neutralised. pr-checks no longer skips the old bot branch names. TechEngine moves from 5577b3f to 3c50cb8: crawlers identify as TechEngine-verify and anti-bot challenges no longer count as alive (TechEngine #127), and photos and logos must be Wikimedia Commons files, with image credits served for every photo category (TechEngine #124, ADR-018). Refs #297 --- .github/workflows/collect.yml | 102 ++++++++++++++++++++++++++++++ .github/workflows/final-check.yml | 49 ++++++++++++-- .github/workflows/pr-checks.yml | 5 -- README.md | 14 +++- TechEngine | 2 +- 5 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/collect.yml diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml new file mode 100644 index 000000000000..be680cfb8d21 --- /dev/null +++ b/.github/workflows/collect.yml @@ -0,0 +1,102 @@ +name: collect + +# Runs one of TechEngine's collectors by hand, on develop: +# ingest drafts new SKUs for one category from upstream catalogs +# refresh re-scrapes the benchmark sources into existing records +# promote runs the network verification (Tiers 1-3) and, with apply, flips +# `verified` on the records that pass +# Each task keeps at most ONE open PR into develop, on a fixed collect/* branch +# that every run force-pushes and whose body it edits in place. Nothing here +# runs on a schedule or posts a comment, and the engine that runs is the +# TechEngine commit this repository pins as its submodule. +# +# The PR comes from github-actions[bot] (this repository's GITHUB_TOKEN), so it +# starts no other workflow: the collector runs the validators and the +# integrity gate before it pushes and puts the result in the PR body. Add the +# `final-check` label for the full build and the Tier 0-3 verification. +# +# Opening the PR needs "Allow GitHub Actions to create and approve pull +# requests" (Settings > Actions > General). The TechEngine workflows are pinned +# by commit SHA; bump it together with the submodule. +on: + workflow_dispatch: + inputs: + task: + description: "Collector to run" + type: choice + options: [ingest, refresh, promote] + default: ingest + category: + description: "ingest: category" + type: choice + options: [cpu, gpu, smartphone] + default: cpu + limit: + description: "ingest: max candidates per source" + type: string + default: "50" + include_drafts: + description: "ingest: also write incomplete records (draft PR)" + type: boolean + default: false + sleep: + description: "refresh: seconds between requests" + type: string + default: "1.0" + apply: + description: "promote: flip verified and open the PR (otherwise a dry run)" + type: boolean + default: false + max_urls: + description: "promote: records to URL-check" + type: string + default: "2000" + max_wikidata: + description: "promote: Wikidata items to check" + type: string + default: "20000" + max_crossref: + description: "promote: records to cross-reference" + type: string + default: "500" + +permissions: {} + +jobs: + ingest: + if: >- + github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' && + inputs.task == 'ingest' + permissions: + contents: write + pull-requests: write + uses: GetTechAPI/TechEngine/.github/workflows/ingest.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124 + with: + category: ${{ inputs.category }} + limit: ${{ inputs.limit }} + include_drafts: ${{ inputs.include_drafts }} + + refresh: + if: >- + github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' && + inputs.task == 'refresh' + permissions: + contents: write + pull-requests: write + uses: GetTechAPI/TechEngine/.github/workflows/refresh.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124 + with: + sleep: ${{ inputs.sleep }} + + promote: + if: >- + github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' && + inputs.task == 'promote' + permissions: + contents: write + pull-requests: write + uses: GetTechAPI/TechEngine/.github/workflows/verify-promote.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124 + with: + apply: ${{ inputs.apply }} + max_urls: ${{ inputs.max_urls }} + max_wikidata: ${{ inputs.max_wikidata }} + max_crossref: ${{ inputs.max_crossref }} diff --git a/.github/workflows/final-check.yml b/.github/workflows/final-check.yml index 48221ca01969..ba1f711dd7fb 100644 --- a/.github/workflows/final-check.yml +++ b/.github/workflows/final-check.yml @@ -2,9 +2,10 @@ name: final-check # A maintainer adds the `final-check` label to an important PR. The check job # builds what pages.yml would publish from the PR (both validators, the -# integrity gate, the whole generated API and the site), and TechEngineBot -# leaves ONE comment with the verdict; adding the label again edits that -# comment. This is the only place the TechEngineBot account is used. +# integrity gate, the whole generated API and the site) and runs the engine's +# Tier 0-3 verification of the records the PR changes. TechEngineBot leaves ONE +# comment with the verdict; adding the label again edits that comment. This is +# the only place the TechEngineBot account is used. # # The check job runs PR code, so it gets no secrets and a read-only token. # Only the comment job sees TECHENGINEBOT_TOKEN, which must be a fine-grained @@ -55,6 +56,27 @@ jobs: tail -n 40 "$RUNNER_TEMP/install.log" exit "$status" + # The only step here that goes online. Tiers 1 and 2 fetch the source URLs + # of at most 40 changed records and look them up on Wikidata, with the + # engine's own bot User-Agent; Tier 3 is a dry run that writes nothing. + # Informational: the verdict doesn't depend on it. + - name: Verify the changed records + id: verify + if: steps.install.outcome == 'success' + continue-on-error: true + timeout-minutes: 20 + working-directory: TechEngine + env: + TECHAPI_DATA_DIR: ${{ github.workspace }}/data + PYTHONDONTWRITEBYTECODE: "1" + run: | + status=0 + python -m app.verify pr --base "$(git -C .. rev-parse HEAD^1)" \ + > "$RUNNER_TEMP/verify.md" 2> "$RUNNER_TEMP/verify.log" || status=$? + cat "$RUNNER_TEMP/verify.md" + tail -n 20 "$RUNNER_TEMP/verify.log" + exit "$status" + - name: Validate the data and generate the API id: api if: steps.install.outcome == 'success' @@ -83,13 +105,17 @@ jobs: exit "$status" # The comment quotes tool output, which can carry text from the PR's - # records: it stays inside code fences with backticks and @ removed, so - # it can neither break out into markdown nor mention anyone. + # records: logs stay inside code fences with backticks and @ removed, so + # they can neither break out into markdown nor mention anyone. The + # verification report keeps its markdown tables, but loses every @ and has + # a zero-width space put into #123, GH-123 and links, so it can't mention + # anyone or cross-reference another issue either. - name: Report id: report if: always() env: INSTALL: ${{ steps.install.outcome }} + VERIFY: ${{ steps.verify.outcome }} API: ${{ steps.api.outcome }} SITE: ${{ steps.site.outcome }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -101,6 +127,9 @@ jobs: tail -n "$2" "$1" 2>/dev/null | clean echo '```' } + zw=$(printf '\342\200\213') + ref="s/(#|[Gg][Hh]-)([0-9]+)(\$|[^0-9A-Za-z_])/\1$zw\2\3/g" # not #3fb950, a chart colour + md_clean() { tr -d '@' | sed -E -e "$ref" -e "$ref" -e "s#://#:$zw//#g" -e "s/www\./www$zw./g" | cut -c1-300; } verdict='✅ Final check passed' [ "$INSTALL/$API/$SITE" = success/success/success ] || verdict='❌ Final check failed' engine=$(git rev-parse --short HEAD:TechEngine 2>/dev/null || echo '?') @@ -115,10 +144,20 @@ jobs: echo "| Install the pinned TechEngine | ${INSTALL:-skipped} |" echo "| Validators, integrity gate, generated API | ${API:-skipped} |" echo "| Site build | ${SITE:-skipped} |" + echo "| Data verification, Tiers 0-3 (informational) | ${VERIFY:-skipped} |" if [ -s "$RUNNER_TEMP/api.log" ]; then echo sed -n '/^| Collection /,/^$/p' "$RUNNER_TEMP/api.log" | tr -d '`@' fi + if [ -s "$RUNNER_TEMP/verify.md" ]; then + echo + echo '
Data verification, Tiers 0-3' + echo + md_clean < "$RUNNER_TEMP/verify.md" | head -c 30000 + echo + echo '
' + fi + [ "$VERIFY" = failure ] && { echo; echo '
Verification log'; echo; fence "$RUNNER_TEMP/verify.log" 20; echo '
'; } [ "$INSTALL" = failure ] && { echo; echo '
Install log'; echo; fence "$RUNNER_TEMP/install.log" 20; echo '
'; } [ "$API" = failure ] && { echo; echo '
Validation and generation log'; echo; fence "$RUNNER_TEMP/api.log" 30; echo '
'; } [ "$SITE" = failure ] && { echo; echo '
Site build log'; echo; fence "$RUNNER_TEMP/site.log" 20; echo '
'; } diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 930ac895e6ab..e0af26fba5fa 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -18,11 +18,6 @@ concurrency: jobs: validate: - # Branches the old TechEngine bots pushed. Nothing should open them any more; - # if something does, it gets no run here. - if: >- - !startsWith(github.head_ref, 'refresh/') && !startsWith(github.head_ref, 'ingest/') && - !startsWith(github.head_ref, 'verify/promote-') && !startsWith(github.head_ref, 'dump-refresh/') runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: diff --git a/README.md b/README.md index cd0be524c08a..278e765e7c92 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,18 @@ comments. For an important PR, a maintainer adds the `final-check` label. [`final-check.yml`](.github/workflows/final-check.yml) then builds everything -`pages.yml` would publish from the PR, and TechEngineBot posts one comment with -the verdict; adding the label again updates that comment. +`pages.yml` would publish from the PR and runs TechEngine's Tier 0-3 +verification of the changed records. TechEngineBot posts one comment with the +verdict; adding the label again updates that comment. + +New and refreshed records come from +[`collect.yml`](.github/workflows/collect.yml), which a maintainer runs by hand +from the Actions tab on `develop`. It runs one of TechEngine's collectors +(`ingest`, `refresh` or `promote`) and keeps one open PR per task on a fixed +`collect/*` branch, updated in place by later runs. That PR comes from +github-actions[bot] and starts no checks of its own, so the collector validates +before it pushes and writes the result into the PR body. Collection never runs +on a schedule and never posts comments. ## Branching diff --git a/TechEngine b/TechEngine index 5577b3f171b9..3c50cb80eb9b 160000 --- a/TechEngine +++ b/TechEngine @@ -1 +1 @@ -Subproject commit 5577b3f171b981b0d2356017b1e5dd445073e996 +Subproject commit 3c50cb80eb9b5e2eb293e23327af707a4b4c82f5