diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml
new file mode 100644
index 000000000000..be680cfb8d21
--- /dev/null
+++ b/.github/workflows/collect.yml
@@ -0,0 +1,102 @@
+name: collect
+
+# Runs one of TechEngine's collectors by hand, on develop:
+# ingest drafts new SKUs for one category from upstream catalogs
+# refresh re-scrapes the benchmark sources into existing records
+# promote runs the network verification (Tiers 1-3) and, with apply, flips
+# `verified` on the records that pass
+# Each task keeps at most ONE open PR into develop, on a fixed collect/* branch
+# that every run force-pushes and whose body it edits in place. Nothing here
+# runs on a schedule or posts a comment, and the engine that runs is the
+# TechEngine commit this repository pins as its submodule.
+#
+# The PR comes from github-actions[bot] (this repository's GITHUB_TOKEN), so it
+# starts no other workflow: the collector runs the validators and the
+# integrity gate before it pushes and puts the result in the PR body. Add the
+# `final-check` label for the full build and the Tier 0-3 verification.
+#
+# Opening the PR needs "Allow GitHub Actions to create and approve pull
+# requests" (Settings > Actions > General). The TechEngine workflows are pinned
+# by commit SHA; bump it together with the submodule.
+on:
+ workflow_dispatch:
+ inputs:
+ task:
+ description: "Collector to run"
+ type: choice
+ options: [ingest, refresh, promote]
+ default: ingest
+ category:
+ description: "ingest: category"
+ type: choice
+ options: [cpu, gpu, smartphone]
+ default: cpu
+ limit:
+ description: "ingest: max candidates per source"
+ type: string
+ default: "50"
+ include_drafts:
+ description: "ingest: also write incomplete records (draft PR)"
+ type: boolean
+ default: false
+ sleep:
+ description: "refresh: seconds between requests"
+ type: string
+ default: "1.0"
+ apply:
+ description: "promote: flip verified and open the PR (otherwise a dry run)"
+ type: boolean
+ default: false
+ max_urls:
+ description: "promote: records to URL-check"
+ type: string
+ default: "2000"
+ max_wikidata:
+ description: "promote: Wikidata items to check"
+ type: string
+ default: "20000"
+ max_crossref:
+ description: "promote: records to cross-reference"
+ type: string
+ default: "500"
+
+permissions: {}
+
+jobs:
+ ingest:
+ if: >-
+ github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' &&
+ inputs.task == 'ingest'
+ permissions:
+ contents: write
+ pull-requests: write
+ uses: GetTechAPI/TechEngine/.github/workflows/ingest.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124
+ with:
+ category: ${{ inputs.category }}
+ limit: ${{ inputs.limit }}
+ include_drafts: ${{ inputs.include_drafts }}
+
+ refresh:
+ if: >-
+ github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' &&
+ inputs.task == 'refresh'
+ permissions:
+ contents: write
+ pull-requests: write
+ uses: GetTechAPI/TechEngine/.github/workflows/refresh.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124
+ with:
+ sleep: ${{ inputs.sleep }}
+
+ promote:
+ if: >-
+ github.repository == 'GetTechAPI/TechAPI' && github.ref == 'refs/heads/develop' &&
+ inputs.task == 'promote'
+ permissions:
+ contents: write
+ pull-requests: write
+ uses: GetTechAPI/TechEngine/.github/workflows/verify-promote.yml@3c50cb80eb9b5e2eb293e23327af707a4b4c82f5 # main, TechEngine #124
+ with:
+ apply: ${{ inputs.apply }}
+ max_urls: ${{ inputs.max_urls }}
+ max_wikidata: ${{ inputs.max_wikidata }}
+ max_crossref: ${{ inputs.max_crossref }}
diff --git a/.github/workflows/final-check.yml b/.github/workflows/final-check.yml
index 48221ca01969..ba1f711dd7fb 100644
--- a/.github/workflows/final-check.yml
+++ b/.github/workflows/final-check.yml
@@ -2,9 +2,10 @@ name: final-check
# A maintainer adds the `final-check` label to an important PR. The check job
# builds what pages.yml would publish from the PR (both validators, the
-# integrity gate, the whole generated API and the site), and TechEngineBot
-# leaves ONE comment with the verdict; adding the label again edits that
-# comment. This is the only place the TechEngineBot account is used.
+# integrity gate, the whole generated API and the site) and runs the engine's
+# Tier 0-3 verification of the records the PR changes. TechEngineBot leaves ONE
+# comment with the verdict; adding the label again edits that comment. This is
+# the only place the TechEngineBot account is used.
#
# The check job runs PR code, so it gets no secrets and a read-only token.
# Only the comment job sees TECHENGINEBOT_TOKEN, which must be a fine-grained
@@ -55,6 +56,27 @@ jobs:
tail -n 40 "$RUNNER_TEMP/install.log"
exit "$status"
+ # The only step here that goes online. Tiers 1 and 2 fetch the source URLs
+ # of at most 40 changed records and look them up on Wikidata, with the
+ # engine's own bot User-Agent; Tier 3 is a dry run that writes nothing.
+ # Informational: the verdict doesn't depend on it.
+ - name: Verify the changed records
+ id: verify
+ if: steps.install.outcome == 'success'
+ continue-on-error: true
+ timeout-minutes: 20
+ working-directory: TechEngine
+ env:
+ TECHAPI_DATA_DIR: ${{ github.workspace }}/data
+ PYTHONDONTWRITEBYTECODE: "1"
+ run: |
+ status=0
+ python -m app.verify pr --base "$(git -C .. rev-parse HEAD^1)" \
+ > "$RUNNER_TEMP/verify.md" 2> "$RUNNER_TEMP/verify.log" || status=$?
+ cat "$RUNNER_TEMP/verify.md"
+ tail -n 20 "$RUNNER_TEMP/verify.log"
+ exit "$status"
+
- name: Validate the data and generate the API
id: api
if: steps.install.outcome == 'success'
@@ -83,13 +105,17 @@ jobs:
exit "$status"
# The comment quotes tool output, which can carry text from the PR's
- # records: it stays inside code fences with backticks and @ removed, so
- # it can neither break out into markdown nor mention anyone.
+ # records: logs stay inside code fences with backticks and @ removed, so
+ # they can neither break out into markdown nor mention anyone. The
+ # verification report keeps its markdown tables, but loses every @ and has
+ # a zero-width space put into #123, GH-123 and links, so it can't mention
+ # anyone or cross-reference another issue either.
- name: Report
id: report
if: always()
env:
INSTALL: ${{ steps.install.outcome }}
+ VERIFY: ${{ steps.verify.outcome }}
API: ${{ steps.api.outcome }}
SITE: ${{ steps.site.outcome }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
@@ -101,6 +127,9 @@ jobs:
tail -n "$2" "$1" 2>/dev/null | clean
echo '```'
}
+ zw=$(printf '\342\200\213')
+ ref="s/(#|[Gg][Hh]-)([0-9]+)(\$|[^0-9A-Za-z_])/\1$zw\2\3/g" # not #3fb950, a chart colour
+ md_clean() { tr -d '@' | sed -E -e "$ref" -e "$ref" -e "s#://#:$zw//#g" -e "s/www\./www$zw./g" | cut -c1-300; }
verdict='✅ Final check passed'
[ "$INSTALL/$API/$SITE" = success/success/success ] || verdict='❌ Final check failed'
engine=$(git rev-parse --short HEAD:TechEngine 2>/dev/null || echo '?')
@@ -115,10 +144,20 @@ jobs:
echo "| Install the pinned TechEngine | ${INSTALL:-skipped} |"
echo "| Validators, integrity gate, generated API | ${API:-skipped} |"
echo "| Site build | ${SITE:-skipped} |"
+ echo "| Data verification, Tiers 0-3 (informational) | ${VERIFY:-skipped} |"
if [ -s "$RUNNER_TEMP/api.log" ]; then
echo
sed -n '/^| Collection /,/^$/p' "$RUNNER_TEMP/api.log" | tr -d '`@'
fi
+ if [ -s "$RUNNER_TEMP/verify.md" ]; then
+ echo
+ echo 'Data verification, Tiers 0-3
'
+ echo
+ md_clean < "$RUNNER_TEMP/verify.md" | head -c 30000
+ echo
+ echo 'Verification log
'; echo; fence "$RUNNER_TEMP/verify.log" 20; echo 'Install log
'; echo; fence "$RUNNER_TEMP/install.log" 20; echo 'Validation and generation log
'; echo; fence "$RUNNER_TEMP/api.log" 30; echo 'Site build log
'; echo; fence "$RUNNER_TEMP/site.log" 20; echo '