From 86a9a2a1d0a298dadb0934acee740d5d7e14ea80 Mon Sep 17 00:00:00 2001 From: Haseeb Malik Date: Thu, 16 Jul 2026 15:20:41 -0400 Subject: [PATCH] Add end-to-end testing workflow, fixtures, and sample source --- .github/workflows/e2e.yml | 209 ++++++++++++++++++++++++++++++++++ .gitignore | 1 + code.json | 63 ++++++++++ e2e/fixtures/code.json.broken | 63 ++++++++++ e2e/fixtures/code.json.good | 63 ++++++++++ e2e/sample/textstats.py | 94 +++++++++++++++ 6 files changed, 493 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 .gitignore create mode 100644 code.json create mode 100644 e2e/fixtures/code.json.broken create mode 100644 e2e/fixtures/code.json.good create mode 100644 e2e/sample/textstats.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..a7e2d76 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,209 @@ +name: E2E +run-name: e2e ${{ inputs.correlation_id }} + +on: + workflow_dispatch: + inputs: + action_sha: + description: "automated-codejson-generator commit to test" + required: true + type: string + pr_number: + description: "The pull request that commit came from" + required: true + type: string + correlation_id: + description: "Lets the caller find this run by name" + required: true + type: string + +# One sandbox, so runs have to take turns or they fight over main and each other's +# pull requests. Queue instead of cancelling so cleanup always gets to finish. +concurrency: + group: e2e-sandbox + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 25 + + steps: + - name: Checkout sandbox + uses: actions/checkout@v4 + + - name: Checkout action under test + uses: actions/checkout@v4 + with: + repository: DSACMS/automated-codejson-generator + ref: ${{ inputs.action_sha }} + path: .e2e-action + + - name: Link back to the pull request + run: | + echo "Testing [PR #${{ inputs.pr_number }}](https://github.com/DSACMS/automated-codejson-generator/pull/${{ inputs.pr_number }}) at \`${{ inputs.action_sha }}\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Stage a valid code.json + run: cp e2e/fixtures/code.json.good code.json + + - name: Validate the good code.json + uses: ./.e2e-action + env: + # The action validates instead of generating when it sees a pull_request + # event. That is the only switch for this mode, so we set it directly. + GITHUB_EVENT_NAME: pull_request + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + + - name: Stage a broken code.json + run: cp e2e/fixtures/code.json.broken code.json + + - name: Validate the broken code.json + id: validate_broken + continue-on-error: true + uses: ./.e2e-action + env: + GITHUB_EVENT_NAME: pull_request + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + + - name: Assert the broken code.json was rejected + run: | + set -euo pipefail + if [ "${{ steps.validate_broken.outcome }}" != "failure" ]; then + echo "::error::The action accepted a code.json that violates the schema." + exit 1 + fi + echo "Rejected, as expected." + + - name: Restore a valid code.json + run: cp e2e/fixtures/code.json.good code.json + + - name: Generate and open a pull request + id: pr_mode + uses: ./.e2e-action + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + SKIP_PR: "false" + + - name: Assert a pull request was opened + id: pr_assert + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_URL: ${{ steps.pr_mode.outputs.pr_url }} + run: | + set -euo pipefail + + # sendPR logs its failures and returns normally, so the action exits 0 even + # when it never opened anything. The output is the only real evidence. + if [ -z "${PR_URL:-}" ]; then + echo "::error::No pr_url output, so no pull request was opened." + exit 1 + fi + + PR_NUMBER="${PR_URL##*/}" + HEAD_REF=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefName --jq .headRefName) + echo "Opened $PR_URL from $HEAD_REF" + + gh api "repos/${GITHUB_REPOSITORY}/contents/code.json?ref=${HEAD_REF}" --jq .content \ + | base64 -d > code.json + + - name: Assert the generated code.json looks right + run: | + set -euo pipefail + jq -e '.name == "codejson-action-e2e"' code.json > /dev/null + jq -e '.repositoryURL == "https://github.com/DSACMS/codejson-action-e2e"' code.json > /dev/null + jq -e '.laborHours > 0' code.json > /dev/null + jq -e '.date.metadataLastUpdated | length > 0' code.json > /dev/null + + - name: Validate the generated code.json + uses: ./.e2e-action + env: + GITHUB_EVENT_NAME: pull_request + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + + - name: Restore a valid code.json + run: cp e2e/fixtures/code.json.good code.json + + - name: Generate and push straight to main + id: push_mode + uses: ./.e2e-action + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Only needs write access to this repo, which the job token already has. + ADMIN_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + SKIP_PR: "true" + + - name: Assert the commit landed on main + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + METHOD: ${{ steps.push_mode.outputs.method_used }} + COMMIT_SHA: ${{ steps.push_mode.outputs.commit_sha }} + run: | + set -euo pipefail + + # pushDirectlyWithFallback quietly opens a pull request when the push fails, + # so exiting 0 doesn't mean anything landed on main. + if [ "${METHOD:-}" != "direct_push" ]; then + echo "::error::Expected a direct push, got '${METHOD:-}'." + exit 1 + fi + + gh api "repos/${GITHUB_REPOSITORY}/contents/code.json?ref=main" --jq .content \ + | base64 -d > code.json + + MAIN_SHA=$(gh api "repos/${GITHUB_REPOSITORY}/commits/main" --jq .sha) + if [ "$MAIN_SHA" != "$COMMIT_SHA" ]; then + echo "::error::main is at ${MAIN_SHA}, but the action reported ${COMMIT_SHA}." + exit 1 + fi + echo "Pushed ${COMMIT_SHA} to main." + + - name: Validate the pushed code.json + uses: ./.e2e-action + env: + GITHUB_EVENT_NAME: pull_request + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: main + + - name: Reset the sandbox + if: always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # No -e here. Every reset should be attempted even if an earlier one fails. + set -uo pipefail + + # Sweep by pattern rather than tracking what this run made, so a run that + # died halfway through still gets cleaned up by the next one. + for BRANCH in $(gh api "repos/${GITHUB_REPOSITORY}/branches?per_page=100" --jq '.[].name' | grep '^code-json-'); do + PR_NUMBER=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq '.[0].number // empty') + if [ -n "$PR_NUMBER" ]; then + gh pr close "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --comment "Closing e2e artifact." + fi + gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/heads/${BRANCH}" + echo "Removed $BRANCH" + done + + BASELINE=$(cat e2e/fixtures/code.json.good) + CURRENT=$(gh api "repos/${GITHUB_REPOSITORY}/contents/code.json?ref=main" --jq .content | base64 -d) + + if [ "$BASELINE" != "$CURRENT" ]; then + gh api -X PUT "repos/${GITHUB_REPOSITORY}/contents/code.json" \ + -f message="Reset sandbox code.json after e2e" \ + -f content="$(base64 -w0 e2e/fixtures/code.json.good)" \ + -f branch=main \ + -f sha="$(gh api "repos/${GITHUB_REPOSITORY}/contents/code.json?ref=main" --jq .sha)" + echo "Reset code.json on main" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ecde58d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.e2e-action/ diff --git a/code.json b/code.json new file mode 100644 index 0000000..a2ff1ec --- /dev/null +++ b/code.json @@ -0,0 +1,63 @@ +{ + "name": "codejson-action-e2e", + "version": "1.0.0", + "description": "Sandbox repository used to end-to-end test the automated code.json generator", + "longDescription": "This repository is a disposable sandbox that exists purely so the automated-codejson-generator action can be exercised for real against a live GitHub repository. It holds a small sample source tree so that scc has something to measure, a known good code.json, and a deliberately invalid one. The end-to-end workflow runs the action here in each of its modes and resets this repository afterwards.", + "status": "Development", + "permissions": { + "licenses": [ + { + "name": "CC0-1.0", + "URL": "https://creativecommons.org/publicdomain/zero/1.0/" + } + ], + "usageType": ["openSource"], + "exemptionText": "" + }, + "organization": "Centers for Medicare & Medicaid Services", + "repositoryURL": "https://github.com/DSACMS/codejson-action-e2e", + "repositoryHost": "github.com/DSACMS", + "repositoryVisibility": "public", + "homepageURL": "", + "downloadURL": "", + "disclaimerURL": "", + "disclaimerText": "", + "vcs": "git", + "laborHours": 0, + "reuseFrequency": { + "forks": 0, + "clones": 0 + }, + "platforms": ["web"], + "categories": ["developer-tools"], + "softwareType": "standalone/backend", + "languages": ["Python"], + "maintenance": "internal", + "contractNumber": [], + "SBOM": "https://github.com/DSACMS/codejson-action-e2e/network/dependencies", + "relatedCode": [], + "reusedCode": [], + "partners": [], + "date": { + "created": "2026-07-15T00:00:00Z", + "lastModified": "2026-07-15T00:00:00Z", + "metadataLastUpdated": "2026-07-15T00:00:00Z" + }, + "tags": ["codejson", "testing"], + "contact": { + "email": "opensource@cms.hhs.gov", + "name": "CMS Open Source Team" + }, + "feedbackMechanism": "https://github.com/DSACMS/codejson-action-e2e/issues", + "AIUseCaseID": "0", + "localisation": false, + "repositoryType": "tools", + "userInput": false, + "fismaLevel": "low", + "group": "CMS/OA/DSAC", + "projects": ["share-it-act"], + "systems": [], + "subsetInHealthcare": ["operational"], + "userType": ["government"], + "maturityModelTier": 1 +} diff --git a/e2e/fixtures/code.json.broken b/e2e/fixtures/code.json.broken new file mode 100644 index 0000000..7be3e88 --- /dev/null +++ b/e2e/fixtures/code.json.broken @@ -0,0 +1,63 @@ +{ + "name": "codejson-action-e2e", + "version": "1.0.0", + "description": "Sandbox repository used to end-to-end test the automated code.json generator", + "longDescription": "Too short to satisfy the schema.", + "status": "Sandbox", + "permissions": { + "licenses": [ + { + "name": "CC0-1.0", + "URL": "https://creativecommons.org/publicdomain/zero/1.0/" + } + ], + "usageType": ["openSource"], + "exemptionText": "" + }, + "organization": "Centers for Medicare & Medicaid Services", + "repositoryURL": "https://github.com/DSACMS/codejson-action-e2e", + "repositoryHost": "github.com/DSACMS", + "repositoryVisibility": "public", + "homepageURL": "", + "downloadURL": "", + "disclaimerURL": "", + "disclaimerText": "", + "vcs": "git", + "laborHours": 0, + "reuseFrequency": { + "forks": 0, + "clones": 0 + }, + "platforms": ["web"], + "categories": ["developer-tools"], + "softwareType": "standalone/backend", + "languages": ["Python"], + "maintenance": "internal", + "contractNumber": [], + "SBOM": "https://github.com/DSACMS/codejson-action-e2e/network/dependencies", + "relatedCode": [], + "reusedCode": [], + "partners": [], + "date": { + "created": "2026-07-15T00:00:00Z", + "lastModified": "2026-07-15T00:00:00Z", + "metadataLastUpdated": "2026-07-15T00:00:00Z" + }, + "tags": ["codejson", "testing"], + "contact": { + "email": "opensource@cms.hhs.gov", + "name": "CMS Open Source Team" + }, + "feedbackMechanism": "https://github.com/DSACMS/codejson-action-e2e/issues", + "AIUseCaseID": "0", + "localisation": false, + "repositoryType": "tools", + "userInput": false, + "fismaLevel": "low", + "group": "CMS/OA/DSAC", + "projects": ["share-it-act"], + "systems": [], + "subsetInHealthcare": ["operational"], + "userType": ["government"], + "maturityModelTier": 1 +} diff --git a/e2e/fixtures/code.json.good b/e2e/fixtures/code.json.good new file mode 100644 index 0000000..a2ff1ec --- /dev/null +++ b/e2e/fixtures/code.json.good @@ -0,0 +1,63 @@ +{ + "name": "codejson-action-e2e", + "version": "1.0.0", + "description": "Sandbox repository used to end-to-end test the automated code.json generator", + "longDescription": "This repository is a disposable sandbox that exists purely so the automated-codejson-generator action can be exercised for real against a live GitHub repository. It holds a small sample source tree so that scc has something to measure, a known good code.json, and a deliberately invalid one. The end-to-end workflow runs the action here in each of its modes and resets this repository afterwards.", + "status": "Development", + "permissions": { + "licenses": [ + { + "name": "CC0-1.0", + "URL": "https://creativecommons.org/publicdomain/zero/1.0/" + } + ], + "usageType": ["openSource"], + "exemptionText": "" + }, + "organization": "Centers for Medicare & Medicaid Services", + "repositoryURL": "https://github.com/DSACMS/codejson-action-e2e", + "repositoryHost": "github.com/DSACMS", + "repositoryVisibility": "public", + "homepageURL": "", + "downloadURL": "", + "disclaimerURL": "", + "disclaimerText": "", + "vcs": "git", + "laborHours": 0, + "reuseFrequency": { + "forks": 0, + "clones": 0 + }, + "platforms": ["web"], + "categories": ["developer-tools"], + "softwareType": "standalone/backend", + "languages": ["Python"], + "maintenance": "internal", + "contractNumber": [], + "SBOM": "https://github.com/DSACMS/codejson-action-e2e/network/dependencies", + "relatedCode": [], + "reusedCode": [], + "partners": [], + "date": { + "created": "2026-07-15T00:00:00Z", + "lastModified": "2026-07-15T00:00:00Z", + "metadataLastUpdated": "2026-07-15T00:00:00Z" + }, + "tags": ["codejson", "testing"], + "contact": { + "email": "opensource@cms.hhs.gov", + "name": "CMS Open Source Team" + }, + "feedbackMechanism": "https://github.com/DSACMS/codejson-action-e2e/issues", + "AIUseCaseID": "0", + "localisation": false, + "repositoryType": "tools", + "userInput": false, + "fismaLevel": "low", + "group": "CMS/OA/DSAC", + "projects": ["share-it-act"], + "systems": [], + "subsetInHealthcare": ["operational"], + "userType": ["government"], + "maturityModelTier": 1 +} diff --git a/e2e/sample/textstats.py b/e2e/sample/textstats.py new file mode 100644 index 0000000..42116d4 --- /dev/null +++ b/e2e/sample/textstats.py @@ -0,0 +1,94 @@ +"""Sample code. It exists so scc has real source to measure.""" + +import re +from collections import Counter + +WORD = re.compile(r"[a-z']+") + +STOP_WORDS = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "but", + "by", + "for", + "if", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "to", + "was", + "were", + "with", +} + + +def words(text): + return WORD.findall(text.lower()) + + +def word_count(text): + return len(words(text)) + + +def unique_words(text): + return set(words(text)) + + +def frequencies(text, skip_stop_words=True): + counts = Counter(words(text)) + if skip_stop_words: + for stop in STOP_WORDS: + counts.pop(stop, None) + return counts + + +def most_common(text, limit=10): + return frequencies(text).most_common(limit) + + +def average_word_length(text): + found = words(text) + if not found: + return 0.0 + return sum(len(word) for word in found) / len(found) + + +def sentences(text): + parts = re.split(r"[.!?]+", text) + return [part.strip() for part in parts if part.strip()] + + +def average_sentence_length(text): + found = sentences(text) + if not found: + return 0.0 + return sum(word_count(part) for part in found) / len(found) + + +def lexical_diversity(text): + total = word_count(text) + if total == 0: + return 0.0 + return len(unique_words(text)) / total + + +def summarize(text): + return { + "words": word_count(text), + "unique": len(unique_words(text)), + "sentences": len(sentences(text)), + "average_word_length": round(average_word_length(text), 2), + "average_sentence_length": round(average_sentence_length(text), 2), + "lexical_diversity": round(lexical_diversity(text), 3), + "top_words": most_common(text, 5), + }