diff --git a/.github/workflows/manual_regenerate_models.yaml b/.github/workflows/manual_regenerate_models.yaml deleted file mode 100644 index 42f48835..00000000 --- a/.github/workflows/manual_regenerate_models.yaml +++ /dev/null @@ -1,257 +0,0 @@ -# This workflow regenerates Pydantic models, TypedDicts, and Literal aliases (src/apify_client/_{models,typeddicts,literals}.py) from the OpenAPI spec. -# -# It can be triggered in two ways: -# 1. Automatically via workflow_dispatch from the apify-docs CI pipeline. -# 2. Manually from the GitHub UI (without any inputs) to regenerate from the live published spec. - -name: Regenerate models - -on: - workflow_dispatch: - inputs: - docs_pr_number: - description: PR number in apify/apify-docs that triggered this workflow (optional for manual runs) - required: false - type: string - docs_workflow_run_id: - description: Workflow run ID in apify/apify-docs that built the OpenAPI spec artifact (optional for manual runs) - required: false - type: string - docs_pr_author: - description: GitHub login of the apify-docs PR author (optional for manual runs) - required: false - type: string - -permissions: - contents: write - pull-requests: write - -concurrency: - group: regenerate-models-${{ inputs.docs_pr_number || 'manual' }} - cancel-in-progress: true - -jobs: - regenerate-models: - name: Regenerate models - runs-on: ubuntu-latest - - env: - DOCS_PR_NUMBER: ${{ inputs.docs_pr_number }} - BRANCH: ${{ inputs.docs_pr_number && format('update-models-docs-pr-{0}', inputs.docs_pr_number) || 'update-models-manual' }} - # Message for the automated regeneration commit. Kept descriptive (and traceable to the docs PR) for the - # branch history. It is deliberately NOT reused as the PR title. - COMMIT_MESSAGE: "${{ inputs.docs_pr_number && format('update generated models from apify-docs PR #{0}', inputs.docs_pr_number) || 'update generated models from published OpenAPI spec' }}" - # Placeholder PR title — just `TODO`, so it carries no valid commit type and `pr-title-check` stays red - # until a human replaces it (the default can never be merged as-is). The apify-docs reference and its - # link live in the PR body, not the title. - PR_TITLE: 'TODO' - ASSIGNEE: ${{ inputs.docs_pr_author || github.actor }} - REVIEWER: vdusek - LABEL: t-tooling - - steps: - - name: Validate inputs - if: inputs.docs_pr_number || inputs.docs_workflow_run_id - env: - DOCS_WORKFLOW_RUN_ID: ${{ inputs.docs_workflow_run_id }} - run: | - if [[ -n "$DOCS_PR_NUMBER" ]] && ! [[ "$DOCS_PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then - echo "::error::docs_pr_number must be a positive integer, got: $DOCS_PR_NUMBER" - exit 1 - fi - if [[ -n "$DOCS_WORKFLOW_RUN_ID" ]] && ! [[ "$DOCS_WORKFLOW_RUN_ID" =~ ^[0-9]+$ ]]; then - echo "::error::docs_workflow_run_id must be a numeric run ID, got: $DOCS_WORKFLOW_RUN_ID" - exit 1 - fi - - - name: Checkout apify-client-python - uses: actions/checkout@v7 - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - # Record master's SHA up front: a later step checks out the auto-update branch, so the change check below - # can't compare against HEAD. Resolve master from the remote rather than the checked-out ref: the run may be - # dispatched from a non-master ref, but that check and the PR both target master, so master is the baseline. - - name: Record master ref - id: base - run: | - git fetch --depth=1 origin master - echo "sha=$(git rev-parse FETCH_HEAD)" >> "$GITHUB_OUTPUT" - - # Does the auto-update branch already exist on the remote? If so, a previous dispatch opened the PR - # and we append to it. If not, we start it from master (below) so signed-commit creates it there. - - name: Determine auto-update branch state - id: branch - run: | - if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then - echo "exists=true" >> "$GITHUB_OUTPUT" - echo "create=false" >> "$GITHUB_OUTPUT" - else - echo "exists=false" >> "$GITHUB_OUTPUT" - echo "create=true" >> "$GITHUB_OUTPUT" - fi - - # Check out the existing branch before regenerating so the new models land as a NEW commit on top, - # mirroring the commit just pushed to the docs PR. We must switch now, while the tree is clean: - # signed-commit's own checkout is a plain `git checkout` that would refuse to overwrite the - # regenerated files (which differ between master and the branch) once they're in the working tree. - - name: Check out existing auto-update branch - if: steps.branch.outputs.exists == 'true' - run: | - git fetch --depth=1 origin "$BRANCH" - git checkout -B "$BRANCH" FETCH_HEAD - - # The branch doesn't exist yet, so regenerate from the recorded master SHA rather than the dispatched - # ref (a manual run may be dispatched from a non-master ref). Regenerating on master keeps the codegen - # tooling current, and signed-commit then creates the branch (the PR head) on top of master, the PR base. - - name: Start the auto-update branch from master - if: steps.branch.outputs.exists == 'false' - run: git checkout "${{ steps.base.outputs.sha }}" - - # Download the pre-built OpenAPI spec artifact from the apify-docs workflow run. - # Skipped for manual runs — datamodel-codegen will fetch from the published spec URL instead. - - name: Download OpenAPI spec artifact - if: inputs.docs_workflow_run_id - uses: actions/download-artifact@v8 - with: - name: openapi-bundles - path: openapi-spec - repository: apify/apify-docs - run-id: ${{ inputs.docs_workflow_run_id }} - github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Set up uv - uses: astral-sh/setup-uv@v9.0.0 - with: - python-version: "3.14" - - - name: Install dependencies - run: uv run poe install-dev - - # When a docs workflow run ID is provided, use the downloaded artifact. - # Otherwise, datamodel-codegen fetches from the default URL configured in pyproject.toml. - - name: Generate models from OpenAPI spec - run: | - if [[ -f openapi-spec/openapi.json ]]; then - uv run poe generate-models-from-file openapi-spec/openapi.json - else - uv run poe generate-models - fi - - # Proceed only when the regenerated models differ from master (compared against the recorded SHA, - # since the tree may now sit on the branch), which skips empty-PR runs: a spec change that doesn't - # affect the client models, or one already merged into master. Also avoids creating an empty branch. - - name: Check for model changes - id: changes - run: | - if git diff --quiet "${{ steps.base.outputs.sha }}" -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then - echo "No model changes relative to master — nothing to regenerate." - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi - - # Append the regenerated models to the auto-update branch as a single signed ("Verified") commit - # via apify/actions/signed-commit (GitHub's createCommitOnBranch GraphQL mutation). It's added on - # top of the branch tip, never resetting to master: a fresh docs PR creates the branch - # (create-branch) and its first commit, and each later docs-PR commit triggers a dispatch that - # appends another, so the client PR mirrors the docs PR and stays open. - # - # Appending (rather than force-pushing the branch to master, as before) is what keeps the PR open: - # a "branch == master" tip makes the PR head equal its base, which GitHub auto-closes, and each - # dispatch then opens a duplicate PR. signed-commit also sets committed=false when the staged - # files match the tip, so a repeated dispatch regenerating identical models adds no commit. - - name: Commit regenerated models - id: commit - if: steps.changes.outputs.has_changes == 'true' - uses: apify/actions/signed-commit@v1.4.0 - with: - message: ${{ env.COMMIT_MESSAGE }} - add: "src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py" - github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - branch: ${{ env.BRANCH }} - create-branch: "${{ steps.branch.outputs.create }}" - - # Ensure exactly one PR exists for this branch. It's no longer auto-closed, so an existing PR is - # reused. Only the first run (or one whose PR step a concurrent dispatch cancelled) creates it. - - name: Create or update PR - if: steps.changes.outputs.has_changes == 'true' - id: pr - env: - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - run: | - EXISTING_PR=$(gh pr list --head "$BRANCH" --json url --jq '.[0].url' 2>/dev/null || true) - - if [[ -n "$EXISTING_PR" ]]; then - echo "PR already exists: $EXISTING_PR" - echo "pr_url=$EXISTING_PR" >> "$GITHUB_OUTPUT" - echo "created=false" >> "$GITHUB_OUTPUT" - else - # The PR opens with a `TODO` placeholder title that fails `pr-title-check`, so it can't be merged - # until a human replaces it with a proper Conventional Commits title. - if [[ -n "$DOCS_PR_NUMBER" ]]; then - DOCS_PR_URL="https://github.com/apify/apify-docs/pull/${DOCS_PR_NUMBER}" - BODY="- Updates the auto-generated Pydantic models and TypedDicts based on the proposed OpenAPI specification changes."$'\n'"- Based on apify-docs PR [#${DOCS_PR_NUMBER}](${DOCS_PR_URL})." - else - BODY="- Updates the auto-generated Pydantic models and TypedDicts based on the latest OpenAPI specification changes."$'\n'"- Based on the [published OpenAPI specification](https://docs.apify.com/api/openapi.json)." - fi - - PR_URL=$(gh pr create \ - --title "$PR_TITLE" \ - --body "$BODY" \ - --head "$BRANCH" \ - --base master \ - --reviewer "$REVIEWER" \ - --assignee "$ASSIGNEE" \ - --label "$LABEL") - echo "Created PR: $PR_URL" - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - echo "created=true" >> "$GITHUB_OUTPUT" - fi - - # Post a cross-repo comment on the docs PR pointing reviewers to the companion client-python PR. - # Only when something happened: the PR was just created, or a commit was appended. A repeated - # dispatch that changes nothing (committed=false) posts no comment, avoiding noise on the docs PR. - - name: Comment on apify-docs PR - if: inputs.docs_pr_number && (steps.pr.outputs.created == 'true' || steps.commit.outputs.committed == 'true') - env: - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - PR_CREATED: ${{ steps.pr.outputs.created }} - PR_URL: ${{ steps.pr.outputs.pr_url }} - DOCS_PR_AUTHOR: ${{ inputs.docs_pr_author }} - run: | - MENTION="" - if [[ -n "$DOCS_PR_AUTHOR" ]]; then - MENTION="@${DOCS_PR_AUTHOR} " - fi - - if [[ "$PR_CREATED" = "true" ]]; then - HEADLINE="A companion PR has been opened in \`apify-client-python\` with the regenerated models: ${PR_URL}" - else - HEADLINE="The companion \`apify-client-python\` PR has been updated with the latest spec changes: ${PR_URL}" - fi - - COMMENT=$(printf '%s\n' \ - "> [!IMPORTANT]" \ - "> **Action required** — ${MENTION}please coordinate this docs PR with the Python API client PR linked below." \ - ">" \ - "> Because this PR modifies the OpenAPI specification, the generated models in \`apify-client-python\` must be regenerated to stay in sync. This has already been done automatically:" \ - ">" \ - "> ${HEADLINE}" \ - ">" \ - "> - Please make sure to review and merge both PRs together to keep the OpenAPI spec and API clients in sync." \ - "> - You can ask for review and help from the Tooling team if needed.") - - gh pr comment "$DOCS_PR_NUMBER" \ - --repo apify/apify-docs \ - --body "$COMMENT" - - - name: Comment on failure - if: failure() && inputs.docs_pr_number - env: - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - run: | - gh pr comment "$DOCS_PR_NUMBER" \ - --repo apify/apify-docs \ - --body "Python client model regeneration failed. [See workflow run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})." \ - || echo "Warning: Failed to post failure comment to apify/apify-docs PR #$DOCS_PR_NUMBER." diff --git a/.github/workflows/on_schedule_regenerate_models.yaml b/.github/workflows/on_schedule_regenerate_models.yaml new file mode 100644 index 00000000..9e854a63 --- /dev/null +++ b/.github/workflows/on_schedule_regenerate_models.yaml @@ -0,0 +1,241 @@ +# Keeps the generated API models in sync with the published OpenAPI specification: every night it regenerates them +# and opens a pull request when the result differs from master. +# +# Two invariants make it safe to run unattended: +# 1. Generation always happens on master, so the output follows the current spec and the current codegen tooling, +# never whatever an older auto-update branch carries. +# 2. The auto-update branch is rebuilt from master instead of appended to, so the diff is always "current spec vs +# current master" and can't resurrect a stale generated file. + +name: Regenerate models + +on: + workflow_dispatch: + + schedule: + - cron: "0 2 * * *" + +concurrency: + group: regenerate-models + cancel-in-progress: false + +# Writes go through the service account token below, not through `GITHUB_TOKEN`. +permissions: + contents: read + +env: + PYTHON_VERSION: 3.14 + BRANCH_NAME: ci/regenerate-models + # Valid Conventional Commits, so the pull request is mergeable as-is; reviewers retitle it to `fix:`/`feat:` when + # the diff is user-facing. + PR_TITLE: "chore: Regenerate models from the published OpenAPI spec" + ASSIGNEE: vdusek + LABEL: t-tooling + +jobs: + regenerate-models: + name: Regenerate models + runs-on: ubuntu-latest + + steps: + - name: Checkout master + uses: actions/checkout@v7 + with: + ref: master + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v9.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: uv run poe install-dev + + # Read before regeneration overwrites it, so the pull request can say whether the specification moved. + - name: Read the recorded specification version + id: previous-spec + run: echo "version=$(uv run python -m scripts.openapi_spec recorded-version)" >> "$GITHUB_OUTPUT" + + # Downloads the specification, generates from it, and records its version in `pyproject.toml`. + - name: Regenerate models + run: uv run poe generate-models + + # Gate on the generated models, not on the specification version: that stamp moves on rebuilds regardless of + # client impact, and doesn't reliably move when the content does (see the pull request body step below). + # Compared against HEAD rather than the index, so nothing staged earlier in the job can hide a change. + - name: Check for model changes + id: changes + run: | + if git diff --quiet HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then + echo "Models are already up to date with the published specification." + echo "has-changes=false" >> "$GITHUB_OUTPUT" + else + git diff --stat HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py + echo "has-changes=true" >> "$GITHUB_OUTPUT" + fi + + # A previous run may already have these exact models up for review; leave it alone instead of churning an open + # pull request. Only the generated files are compared - the branch being behind master says nothing about + # whether the models on it are still the right ones. + - name: Check whether the models are already up for review + id: review + if: steps.changes.outputs.has-changes == 'true' + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + run: | + # `--exit-code` reports 2 for "no such branch"; any other failure must not read as an absent branch, or a + # transient network error would retire a current pull request. + git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1 && status=0 || status=$? + if [[ "$status" == "2" ]]; then + echo "No auto-update branch exists yet." + echo "is-new=true" >> "$GITHUB_OUTPUT" + exit 0 + elif [[ "$status" != "0" ]]; then + echo "Failed to look up branch $BRANCH_NAME on the remote (git ls-remote exited $status)." >&2 + exit 1 + fi + + # The branch only counts as proposed while a pull request is open on it: a leftover branch - a run that + # failed before opening one, or a review that closed it without deleting the branch - must be rebuilt, or + # its contents would look proposed and silently suppress every future regeneration. An empty result counts + # only once the query itself succeeded, otherwise an API error would replace a live pull request. + if ! PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty'); then + echo "Failed to list open pull requests for $BRANCH_NAME." >&2 + exit 1 + fi + + if [[ -z "$PR_NUMBER" ]]; then + echo "Branch $BRANCH_NAME has no open pull request - rebuilding it." + echo "is-new=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git fetch origin "$BRANCH_NAME" + if git diff --quiet FETCH_HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then + echo "The open pull request already carries these models - nothing to do." + echo "is-new=false" >> "$GITHUB_OUTPUT" + else + echo "is-new=true" >> "$GITHUB_OUTPUT" + fi + + # Retire the previous branch before recreating it. Deleting it first also avoids the window where the branch + # tip would equal master, which GitHub reads as an empty pull request and auto-closes. + - name: Retire the superseded pull request + if: steps.changes.outputs.has-changes == 'true' && steps.review.outputs.is-new == 'true' + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty') + + if [[ -n "$PR_NUMBER" ]]; then + gh pr close "$PR_NUMBER" --delete-branch \ + --comment "Superseded by a newer regeneration run: ${RUN_URL}." + echo "Closed superseded PR #${PR_NUMBER}." + elif git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1; then + git push origin --delete "$BRANCH_NAME" + echo "Deleted leftover branch $BRANCH_NAME (no open pull request)." + fi + + # Creates the branch at the current master commit and lands the regenerated files as one signed ("Verified") + # commit, via GitHub's createCommitOnBranch mutation. + - name: Commit the regenerated models + id: commit + if: steps.changes.outputs.has-changes == 'true' && steps.review.outputs.is-new == 'true' + uses: apify/actions/signed-commit@v1.4.0 + with: + message: ${{ env.PR_TITLE }} + add: >- + pyproject.toml + src/apify_client/_models.py + src/apify_client/_typeddicts.py + src/apify_client/_literals.py + github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + branch: ${{ env.BRANCH_NAME }} + create-branch: "true" + + - name: Create the pull request + if: steps.commit.outputs.committed == 'true' + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/workflows/on_schedule_regenerate_models.yaml + PREVIOUS_SPEC_VERSION: ${{ steps.previous-spec.outputs.version }} + run: | + SPEC_VERSION=$(uv run python -m scripts.openapi_spec recorded-version) + + # A moved stamp proves the specification changed; an unchanged one proves nothing, because apify-docs bumps + # `components/version.yaml` in a follow-up `[skip ci]` commit and a deploy can publish new content under + # the old stamp. Say which case this is instead of letting the reviewer read it as a content identity. + if [[ "$SPEC_VERSION" == "$PREVIOUS_SPEC_VERSION" ]]; then + SPEC_LINE="- Specification version: \`${SPEC_VERSION}\` - **unchanged**, which doesn't rule out a specification change: the published stamp can lag its content by a deploy. Only the diff tells." + else + SPEC_LINE="- Specification version: \`${PREVIOUS_SPEC_VERSION}\` -> \`${SPEC_VERSION}\`." + fi + + BODY=$(printf '%s\n' \ + "- Regenerates the Pydantic models, TypedDicts, and literal aliases from the [published OpenAPI specification](https://docs.apify.com/api/openapi.json), and records its version in \`pyproject.toml\`." \ + "${SPEC_LINE}" \ + "" \ + "> [!IMPORTANT]" \ + "> Retitle this pull request to \`fix:\` or \`feat:\` when the diff is user-facing, so that it lands in the changelog and triggers a release - \`chore:\` does neither." \ + "" \ + "> Generated by the [Regenerate models](${WORKFLOW_URL}) workflow.") + + gh pr create \ + --title "$PR_TITLE" \ + --body "$BODY" \ + --base master \ + --head "$BRANCH_NAME" \ + --assignee "$ASSIGNEE" \ + --label "$LABEL" + + # Without this a broken sync stops regeneration silently: GitHub only notifies whoever last touched the cron, and + # no release waits on this workflow. Skipped on manual dispatch so ad-hoc triggers don't spam the channel. + notify_on_failure: + name: Notify Slack on failure + needs: regenerate-models + if: failure() && github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Build Slack payload + env: + REPO: ${{ github.repository }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + HEADING: ":red_circle: Nightly model regeneration failed" + run: | + jq -n \ + --arg repo "${REPO}" \ + --arg url "${WORKFLOW_URL}" \ + --arg heading "${HEADING}" \ + '{ + text: "\($heading) in \($repo)", + blocks: [ + { + type: "header", + text: { type: "plain_text", text: $heading, emoji: true } + }, + { + type: "section", + fields: [ + { type: "mrkdwn", text: "*Repository:*\n\($repo)" }, + { type: "mrkdwn", text: "*Workflow run:*\n<\($url)|View on GitHub>" } + ] + }, + { + type: "section", + text: { type: "mrkdwn", text: "The generated API models are no longer being kept in sync with the published OpenAPI specification." } + } + ] + }' > slack-payload.json + + - name: Send Slack notification + uses: slackapi/slack-github-action@v4.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook + payload-file-path: slack-payload.json diff --git a/.rules.md b/.rules.md index f8407299..27460217 100644 --- a/.rules.md +++ b/.rules.md @@ -19,7 +19,7 @@ uv run poe type-check # Run ty type checker uv run poe unit-tests # Run unit tests uv run poe check-docstrings # Verify async docstrings match sync uv run poe fix-docstrings # Auto-fix async docstrings -uv run poe generate-models # Regenerate _models.py and _typeddicts.py from live OpenAPI spec +uv run poe generate-models # Regenerate _models.py, _typeddicts.py, _literals.py from the published spec uv run poe generate-models-from-file # Regenerate from a local OpenAPI spec file # Run a single test @@ -82,7 +82,11 @@ Each input-side TypedDict ships in two casings: snake_case (`RequestDict`) and c - To regenerate locally: - From the live published spec: `uv run poe generate-models` - From a local spec file: `uv run poe generate-models-from-file path/to/openapi.json` -- In CI, model regeneration is triggered automatically by the `apify/apify-docs` repo when its OpenAPI spec changes (workflow `manual_regenerate_models.yaml`). It downloads the pre-built `openapi-bundles` artifact from the apify-docs workflow run, opens a PR with a placeholder title (`TODO: replace with a Conventional Commits title...`) that the assignee must replace based on the actual model diff, assigns it to the docs PR author, and posts a cross-repo comment on the original apify-docs PR. The apify-docs PR reference lives in the PR body, not the title +- `scripts/openapi_spec.py` downloads the spec once into git-ignored `tmp/openapi.json` — both codegen passes read that copy, so a spec redeployed mid-run can't leave the generated files describing two different inputs — then records its `info.version` in `pyproject.toml` under `[tool.apify.openapi-spec]` +- That version is a **coarse marker, not a content identity**: the spec is served latest-only, and apify-docs bumps `components/version.yaml` in a follow-up `[skip ci]` commit, so a deploy can publish new content under the old stamp. A moved value proves the spec changed; an unchanged value proves nothing +- Recording is unconditional; the nightly workflow keeps it honest by committing nothing unless the models changed. A local `generate-models` that only moves that line should be dropped, not committed alone. `generate-models-from-file` leaves the line alone, so models from a candidate spec must not be committed +- In CI, the `Regenerate models` workflow (`on_schedule_regenerate_models.yaml`) runs nightly at 02:00 UTC and opens a PR when the models change. It always generates on master and rebuilds its `ci/regenerate-models` branch from master instead of appending, so the PR diff is always "current spec vs current master". Retitle the `chore:` PR to `fix:`/`feat:` when the diff is user-facing, so it lands in the changelog and triggers a release +- The gate is the **generated models**, not the spec version — which moves on rebuilds regardless of client impact, and doesn't reliably move when the content does - Manual regeneration is also possible from the GitHub Actions UI (`Regenerate models` workflow) ## Code Conventions diff --git a/pyproject.toml b/pyproject.toml index 25b8fe85..2056e144 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,17 @@ custom_file_header = "# generated by datamodel-codegen" disable_timestamp = true keep_model_order = true +# Which published OpenAPI specification the committed `_models.py`, `_typeddicts.py`, and `_literals.py` were +# generated from. A coarse marker, not a content identity: the specification is served latest-only, so the version +# can't fetch that document back, and apify-docs bumps `components/version.yaml` in a follow-up `[skip ci]` commit, +# so a deploy can publish new content under the old stamp. +# +# `poe generate-models` records whatever it downloaded, unconditionally; what keeps the value honest is the nightly +# workflow committing nothing unless the models changed. A local regeneration that only moves this line is that +# same case - drop it rather than committing it alone. +[tool.apify.openapi-spec] +version = "v2-2026-07-28T083939Z" + [tool.uv] # Minimal defense against supply-chain atatcks. exclude-newer = "24 hours" @@ -293,19 +304,26 @@ cwd = "website" shell = "./build_api_reference.sh && pnpm install && uv run pnpm start" cwd = "website" +# Both passes read one downloaded copy, so a specification redeployed mid-run can't leave the generated files +# describing two different inputs. The version is recorded last, once generation succeeded. +# # The `--alias-generator to_camel` flag lives on the `_models.py` command (not in `[tool.datamodel-codegen]`) # because datamodel-codegen only allows it for `pydantic_v2.BaseModel` output and would reject the TypedDict run. [tool.poe.tasks.generate-models] shell = """ -uv run datamodel-codegen --url https://docs.apify.com/api/openapi.json \ +uv run python -m scripts.openapi_spec fetch \ + && uv run datamodel-codegen --input tmp/openapi.json \ --alias-generator to_camel \ - && uv run datamodel-codegen --url https://docs.apify.com/api/openapi.json \ + && uv run datamodel-codegen --input tmp/openapi.json \ --output src/apify_client/_typeddicts.py \ --output-model-type typing.TypedDict \ --no-use-closed-typed-dict \ - && python scripts/postprocess_generated_models.py + && python scripts/postprocess_generated_models.py \ + && uv run python -m scripts.openapi_spec record-version """ +# For a candidate specification that isn't published yet. The recorded version deliberately stays put - it names +# the published specification - so models generated this way must not be committed, hence the warning. [tool.poe.tasks.generate-models-from-file] shell = """ uv run datamodel-codegen --input $input_file \ @@ -314,6 +332,7 @@ uv run datamodel-codegen --input $input_file \ --output src/apify_client/_typeddicts.py \ --output-model-type typing.TypedDict \ --no-use-closed-typed-dict \ - && python scripts/postprocess_generated_models.py + && python scripts/postprocess_generated_models.py \ + && echo "Generated from $input_file - the version in [tool.apify.openapi-spec] still names the published specification, so do not commit these models." """ args = [{ name = "input-file", positional = true, required = true }] diff --git a/scripts/openapi_spec.py b/scripts/openapi_spec.py new file mode 100644 index 00000000..cd07a0ff --- /dev/null +++ b/scripts/openapi_spec.py @@ -0,0 +1,213 @@ +"""Fetch the published OpenAPI specification and record the version the models were generated from. + +`fetch` downloads the specification into git-ignored `tmp/`, and both codegen passes read that one copy, so a +specification redeployed mid-run can't yield models built from two different inputs. Only the *version* is +committed, in `[tool.apify.openapi-spec]` in `pyproject.toml`. + +`record-version` writes it last, once generation succeeded, so it names the specification the committed +`_models.py`, `_typeddicts.py`, and `_literals.py` follow from. `recorded-version` prints it; the nightly workflow +reads it before regenerating to report whether the stamp moved. + +The stamp is a coarse marker, not a content identity: the specification is served latest-only, so it can't be +fetched back, and apify-docs bumps it in a follow-up `[skip ci]` commit, so a deploy can publish new content under +the old stamp. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +import tomllib +from pathlib import Path +from typing import NoReturn + +import impit + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# The published, bundled specification, built and deployed from the `apify/apify-docs` repository. +SPEC_URL = 'https://docs.apify.com/api/openapi.json' + +# Codegen input, deliberately outside version control - `tmp/` is git-ignored. +SPEC_PATH = REPO_ROOT / 'tmp' / 'openapi.json' + +PYPROJECT_PATH = REPO_ROOT / 'pyproject.toml' + +VERSION_TABLE_PATH = ('tool', 'apify', 'openapi-spec') +VERSION_TABLE = f'[{".".join(VERSION_TABLE_PATH)}]' +VERSION_KEY = 'version' + +# Writing replaces the value in place to keep the surrounding comments. The patterns tolerate the whitespace and +# trailing comments TOML allows, so reformatting `pyproject.toml` can't quietly break the nightly regeneration. +TABLE_HEADER_PATTERN = re.compile(rf'^\s*\[\s*{re.escape(".".join(VERSION_TABLE_PATH))}\s*\]\s*(?:#.*)?$') +ANY_TABLE_HEADER_PATTERN = re.compile(r'^\s*\[') +VERSION_ENTRY_PATTERN = re.compile(rf'^(?P\s*{VERSION_KEY}\s*=\s*)"(?P[^"]*)"(?P.*)$') + +# An error page or a truncated response must never be generated from; the real specification is roughly 1 MB. +MIN_SPEC_SIZE_BYTES = 100_000 + +# Members every specification we can generate models from has to contain; `info` carries the recorded version. +REQUIRED_SPEC_KEYS = ('openapi', 'info', 'paths', 'components') + +REQUEST_TIMEOUT_SECS = 60 + +# The nightly workflow alerts the team when this fails, so a single network blip shouldn't be worth a ping. +DOWNLOAD_ATTEMPTS = 3 +RETRY_DELAY_SECS = 5 + + +def fail(message: str) -> NoReturn: + """Report a failure on stderr and exit non-zero.""" + print(message, file=sys.stderr) + sys.exit(1) + + +def download_spec() -> bytes: + """Download the published specification, retrying transient failures.""" + last_error = '' + + with impit.Client(follow_redirects=True) as client: + for attempt in range(1, DOWNLOAD_ATTEMPTS + 1): + try: + response = client.request('GET', SPEC_URL, timeout=REQUEST_TIMEOUT_SECS) + except Exception as exc: + last_error = f'{type(exc).__name__}: {exc}' + else: + if response.status_code == 200: + return response.content + last_error = f'HTTP {response.status_code}' + + print(f'Attempt {attempt}/{DOWNLOAD_ATTEMPTS} to download {SPEC_URL} failed ({last_error}).') + if attempt < DOWNLOAD_ATTEMPTS: + time.sleep(RETRY_DELAY_SECS) + + fail(f'Failed to download {SPEC_URL} after {DOWNLOAD_ATTEMPTS} attempts: {last_error}.') + + +def read_spec_version(payload: bytes) -> str: + """Validate a downloaded specification and return its `info.version`.""" + if len(payload) < MIN_SPEC_SIZE_BYTES: + fail(f'Downloaded specification is only {len(payload)} bytes, which cannot be the real one - aborting.') + + try: + spec = json.loads(payload) + except json.JSONDecodeError as exc: + fail(f'Downloaded specification is not valid JSON: {exc}.') + + if not isinstance(spec, dict): + fail(f'Downloaded specification is a JSON {type(spec).__name__}, not an object - aborting.') + + missing_keys = [key for key in REQUIRED_SPEC_KEYS if key not in spec] + if missing_keys: + fail(f'Downloaded specification is missing top-level {", ".join(missing_keys)} - aborting.') + + info = spec['info'] + version = info.get(VERSION_KEY) if isinstance(info, dict) else None + if not isinstance(version, str) or not version: + fail('Downloaded specification has no `info.version` string - aborting.') + + return version + + +def fetch() -> None: + """Download the published specification for codegen to read.""" + payload = download_spec() + version = read_spec_version(payload) + + # Written byte for byte, so the key order the generator sees is the published one: `keep_model_order` ties + # the order of the generated models to it. + SPEC_PATH.parent.mkdir(parents=True, exist_ok=True) + SPEC_PATH.write_bytes(payload) + + print(f'Wrote {SPEC_PATH.relative_to(REPO_ROOT)} (version {version}, {len(payload)} bytes).') + + +def read_recorded_version() -> str: + """Return the specification version currently recorded in `pyproject.toml`.""" + try: + config = tomllib.loads(PYPROJECT_PATH.read_text(encoding='utf-8')) + except tomllib.TOMLDecodeError as exc: + fail(f'{PYPROJECT_PATH.name} is not valid TOML: {exc}.') + + for key in VERSION_TABLE_PATH: + if not isinstance(config, dict) or key not in config: + fail(f'{PYPROJECT_PATH.name} has no {VERSION_TABLE} table - cannot read the specification version.') + config = config[key] + + version = config.get(VERSION_KEY) if isinstance(config, dict) else None + if not isinstance(version, str) or not version: + fail(f'{VERSION_TABLE} in {PYPROJECT_PATH.name} has no `{VERSION_KEY}` string.') + + return version + + +def write_recorded_version(version: str) -> None: + """Replace the recorded specification version in `pyproject.toml`, leaving the rest of the file untouched.""" + lines = PYPROJECT_PATH.read_text(encoding='utf-8').splitlines(keepends=True) + + try: + table_index = next(index for index, line in enumerate(lines) if TABLE_HEADER_PATTERN.match(line.rstrip('\n'))) + except StopIteration: + fail(f'{PYPROJECT_PATH.name} has no {VERSION_TABLE} table - cannot record the specification version.') + + # Only the table's own entries may be rewritten, so a missing key can't silently hit the next table's `version`. + for index in range(table_index + 1, len(lines)): + line = lines[index].rstrip('\n') + if ANY_TABLE_HEADER_PATTERN.match(line): + break + + match = VERSION_ENTRY_PATTERN.match(line) + if match: + lines[index] = f'{match["prefix"]}"{version}"{match["suffix"]}\n' + PYPROJECT_PATH.write_text(''.join(lines), encoding='utf-8', newline='\n') + return + + fail(f'{VERSION_TABLE} in {PYPROJECT_PATH.name} has no `{VERSION_KEY}` entry - cannot record the version.') + + +def record_version() -> None: + """Record the fetched specification's version in `pyproject.toml`.""" + if not SPEC_PATH.is_file(): + fail(f'{SPEC_PATH.relative_to(REPO_ROOT)} is missing - run `poe generate-models` instead of this alone.') + + version = read_spec_version(SPEC_PATH.read_bytes()) + previous = read_recorded_version() + + if previous == version: + print(f'Specification version {version} is already recorded in {PYPROJECT_PATH.name}.') + return + + write_recorded_version(version) + print(f'Recorded specification version in {PYPROJECT_PATH.name}: {previous} -> {version}.') + + +def recorded_version() -> None: + """Print the recorded specification version, for the regeneration workflow to read.""" + print(read_recorded_version()) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(required=True) + + fetch_parser = subparsers.add_parser('fetch', help='download the published specification into tmp/') + fetch_parser.set_defaults(handler=fetch) + + record_parser = subparsers.add_parser( + 'record-version', help="record the fetched specification's version in pyproject.toml" + ) + record_parser.set_defaults(handler=record_version) + + show_parser = subparsers.add_parser( + 'recorded-version', help='print the specification version recorded in pyproject.toml' + ) + show_parser.set_defaults(handler=recorded_version) + + parser.parse_args().handler() + + +if __name__ == '__main__': + main() diff --git a/src/apify_client/_models.py b/src/apify_client/_models.py index aad0b44e..7f2b1121 100644 --- a/src/apify_client/_models.py +++ b/src/apify_client/_models.py @@ -2087,9 +2087,6 @@ class Plan(BaseModel): max_actor_task_count: Annotated[int | None, Field(examples=[1000])] = None data_retention_days: Annotated[int | None, Field(examples=[14])] = None available_proxy_groups: dict[str, int] - """ - The number of available proxies in this group. - """ team_account_seat_count: Annotated[int | None, Field(examples=[1])] = None support_level: Annotated[str | None, Field(examples=['COMMUNITY'])] = None available_add_ons: Annotated[list[str] | None, Field(examples=[[]])] = None diff --git a/tests/unit/test_openapi_spec.py b/tests/unit/test_openapi_spec.py new file mode 100644 index 00000000..022a1ffa --- /dev/null +++ b/tests/unit/test_openapi_spec.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from scripts import openapi_spec +from scripts.openapi_spec import ( + MIN_SPEC_SIZE_BYTES, + read_recorded_version, + read_spec_version, + record_version, + write_recorded_version, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def make_spec(**overrides: Any) -> bytes: + """Build a specification payload that passes validation, padded past the minimum size.""" + spec: dict[str, Any] = { + 'openapi': '3.1.2', + 'info': {'title': 'Apify API', 'version': 'v2-2026-07-28T083939Z'}, + 'paths': {}, + 'components': {}, + } + spec.update(overrides) + spec['x-padding'] = 'x' * MIN_SPEC_SIZE_BYTES + return json.dumps(spec).encode() + + +def make_pyproject(tmp_path: Path, content: str) -> Path: + """Write a `pyproject.toml` stub and point the module at it.""" + path = tmp_path / 'pyproject.toml' + path.write_text(content, encoding='utf-8') + return path + + +def test_read_spec_version_returns_the_version() -> None: + """A valid specification yields its `info.version`.""" + assert read_spec_version(make_spec()) == 'v2-2026-07-28T083939Z' + + +@pytest.mark.parametrize( + 'payload', + [ + pytest.param(b'{}', id='too small'), + pytest.param(b'error' + b'x' * MIN_SPEC_SIZE_BYTES, id='not JSON'), + pytest.param(json.dumps(['x' * MIN_SPEC_SIZE_BYTES]).encode(), id='JSON array'), + pytest.param(json.dumps({'padding': 'x' * MIN_SPEC_SIZE_BYTES}).encode(), id='missing top-level members'), + pytest.param(make_spec(info={'title': 'Apify API'}), id='no info.version'), + pytest.param(make_spec(info={'version': ''}), id='empty info.version'), + pytest.param(make_spec(info='not an object'), id='info is not an object'), + ], +) +def test_read_spec_version_rejects_unusable_payloads(payload: bytes) -> None: + """An error page, a truncated response, or a specification without a version stamp aborts instead of generating.""" + with pytest.raises(SystemExit) as exit_info: + read_spec_version(payload) + + assert exit_info.value.code == 1 + + +@pytest.mark.parametrize( + 'content', + [ + pytest.param('[tool.apify.openapi-spec]\nversion = "old"\n', id='canonical'), + pytest.param('[tool.apify.openapi-spec]\nversion="old"\n', id='no spaces around the equals sign'), + pytest.param('[ tool.apify.openapi-spec ]\nversion = "old"\n', id='spaces inside the table header'), + pytest.param('[tool.apify.openapi-spec] # pinned\nversion = "old"\n', id='comment after the table header'), + pytest.param('[tool.apify.openapi-spec]\n# a note\nversion = "old"\n', id='comment before the entry'), + pytest.param('[tool.other]\nversion = "keep"\n\n[tool.apify.openapi-spec]\nversion = "old"\n', id='not first'), + ], +) +def test_write_recorded_version_rewrites_any_valid_layout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, content: str +) -> None: + """Reformatting `pyproject.toml` doesn't break recording, so the nightly regeneration can't be tripped by it.""" + path = make_pyproject(tmp_path, content) + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + + write_recorded_version('new') + + assert read_recorded_version() == 'new' + + +def test_write_recorded_version_preserves_a_trailing_comment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Only the quoted value is replaced, so a comment on the same line survives.""" + path = make_pyproject(tmp_path, '[tool.apify.openapi-spec]\nversion = "old" # bumped by the nightly job\n') + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + + write_recorded_version('new') + + assert path.read_text(encoding='utf-8').endswith('version = "new" # bumped by the nightly job\n') + + +def test_write_recorded_version_leaves_a_later_table_alone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A missing entry aborts rather than falling through to the next table's `version`.""" + path = make_pyproject(tmp_path, '[tool.apify.openapi-spec]\nother = "x"\n\n[tool.next]\nversion = "keep"\n') + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + + with pytest.raises(SystemExit) as exit_info: + write_recorded_version('new') + + assert exit_info.value.code == 1 + assert 'version = "keep"' in path.read_text(encoding='utf-8') + + +def test_write_recorded_version_rejects_a_missing_table(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Without the table there is nowhere to record the version, so the run fails instead of guessing.""" + path = make_pyproject(tmp_path, '[tool.other]\nversion = "keep"\n') + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + + with pytest.raises(SystemExit) as exit_info: + write_recorded_version('new') + + assert exit_info.value.code == 1 + assert path.read_text(encoding='utf-8') == '[tool.other]\nversion = "keep"\n' + + +def test_record_version_writes_the_fetched_version(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The fetched specification's version replaces the recorded one.""" + path = make_pyproject(tmp_path, '[tool.apify.openapi-spec]\nversion = "v2-2020-01-01T000000Z"\n') + spec_path = tmp_path / 'openapi.json' + spec_path.write_bytes(make_spec()) + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + monkeypatch.setattr(openapi_spec, 'SPEC_PATH', spec_path) + + record_version() + + assert read_recorded_version() == 'v2-2026-07-28T083939Z' + + +def test_record_version_is_idempotent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Recording an already-recorded version leaves the file byte for byte identical.""" + path = make_pyproject(tmp_path, '[tool.apify.openapi-spec]\nversion = "v2-2026-07-28T083939Z"\n') + spec_path = tmp_path / 'openapi.json' + spec_path.write_bytes(make_spec()) + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + monkeypatch.setattr(openapi_spec, 'SPEC_PATH', spec_path) + before = path.read_bytes() + + record_version() + + assert path.read_bytes() == before + + +def test_record_version_requires_a_fetched_specification(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Running the recording step on its own aborts instead of recording a stale or absent version.""" + path = make_pyproject(tmp_path, '[tool.apify.openapi-spec]\nversion = "old"\n') + monkeypatch.setattr(openapi_spec, 'REPO_ROOT', tmp_path) + monkeypatch.setattr(openapi_spec, 'PYPROJECT_PATH', path) + monkeypatch.setattr(openapi_spec, 'SPEC_PATH', tmp_path / 'missing.json') + + with pytest.raises(SystemExit) as exit_info: + record_version() + + assert exit_info.value.code == 1 + assert read_recorded_version() == 'old'