From 108fda096b02d2bf450ab6ff994b7004c2ec0d06 Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Sun, 9 Aug 2026 17:32:58 +0200 Subject: [PATCH 1/7] /etc/scripts: Restructure to a single point of truth Previously, version pins for Python, `mypy`, `ruff`, and the API/spec versions lived in multiple places: workflow env blocks, three `pyproject.toml` files, `ruff.toml`, and the server Dockerfiles. `mypy` was pinned in the SDK but unpinned in the server and compliance tool, and `ruff` was not pinned anywhere. Bumping any of these values required editing several files and was easy to get wrong. This change introduces a repo-root `versions.yaml` as the single source of truth. A new `check_global_versions_coincide.py` verifies that every listed `pyproject.toml`, `config.yaml`, and Dockerfile coincides with the yaml. New entries under `dev_tools` are checked automatically without further script changes. A composite action `.github/actions/load-versions` exposes the values to CI jobs as `X_*` environment variables so consuming workflows keep their existing `${{ env.X_* }}` usage. Runtime server constants (currently only `api_base_path`) live in a separate `server/config.yaml`, read by the server directly at import time. The old `check_python_versions_coincide.py` is superseded and removed. Fixes #548 --- .github/actions/load-versions/action.yml | 27 +++ .github/workflows/main.yml | 5 +- .github/workflows/pr.yml | 60 +++--- compliance_tool/pyproject.toml | 2 +- etc/scripts/check_global_versions_coincide.py | 183 ++++++++++++++++++ etc/scripts/check_python_versions_coincide.py | 73 ------- ...ts.txt => check_versions_requirements.txt} | 2 + server/config.yaml | 1 + server/pyproject.toml | 2 +- versions.yaml | 40 ++++ 10 files changed, 280 insertions(+), 115 deletions(-) create mode 100644 .github/actions/load-versions/action.yml create mode 100644 etc/scripts/check_global_versions_coincide.py delete mode 100644 etc/scripts/check_python_versions_coincide.py rename etc/scripts/{check_python_versions_requirements.txt => check_versions_requirements.txt} (57%) create mode 100644 server/config.yaml create mode 100644 versions.yaml diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml new file mode 100644 index 000000000..164bfe29f --- /dev/null +++ b/.github/actions/load-versions/action.yml @@ -0,0 +1,27 @@ +name: load-versions +description: > + Loads values from the repo-root `versions.yaml` into environment variables + for the current job (via `$GITHUB_ENV`). Downstream steps consume via + `${{ env.X_* }}`. Requires `actions/checkout` to have run first. + + Environment variables set: + X_PYTHON_MIN_VERSION (python.min) + X_PYTHON_MAX_VERSION (python.max) + X_MYPY_VERSION (tools.mypy) + X_RUFF_VERSION (tools.ruff) + X_API_VERSION (spec.api_version) + X_AAS_SPECS_RELEASE_TAG (spec.aas) + +runs: + using: composite + steps: + - shell: bash + run: | + { + echo "X_PYTHON_MIN_VERSION=$(yq '.python.min' versions.yaml)" + echo "X_PYTHON_MAX_VERSION=$(yq '.python.max' versions.yaml)" + echo "X_MYPY_VERSION=$(yq '.tools.mypy' versions.yaml)" + echo "X_RUFF_VERSION=$(yq '.tools.ruff' versions.yaml)" + echo "X_API_VERSION=$(yq '.spec.api_version' versions.yaml)" + echo "X_AAS_SPECS_RELEASE_TAG=$(yq '.spec.aas' versions.yaml)" + } >> "$GITHUB_ENV" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3f4f537d..7d61ec909 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,9 +10,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - X_API_VERSION: "v3.1" - jobs: server-docker-arm: # This job checks if we can build our server package @@ -25,6 +22,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions + - uses: ./.github/actions/build-server id: build with: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ce3008e4d..71f84a7e2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -12,11 +12,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - X_PYTHON_MIN_VERSION: "3.10" - X_PYTHON_MAX_VERSION: "3.12" - X_API_VERSION: "v3.1" - jobs: helper-scripts-static-analysis: #This job runs static code analysis with ruff on the helper scripts in ./etc/scripts @@ -26,33 +21,36 @@ jobs: working-directory: ./etc/scripts steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} - name: Install Ruff - # Revisit when Ruff version is specified in single source of truth run: | python -m pip install --upgrade pip - python -m pip install ruff==0.16.0 + python -m pip install ruff==${{ env.X_RUFF_VERSION }} - name: Check formatting and linting rules with Ruff run: | - python -m ruff check + python -m ruff check - check-python-versions: - # This job checks that the Python Versions we support match and are not End of Life + check-global-versions: + # Verifies that (1) the supported Python versions are not EoL and (2) the values + # declared in `versions.yaml` coincide with every module's `pyproject.toml` and + # any listed `config.yaml` file. runs-on: ubuntu-latest defaults: run: working-directory: ./etc/scripts steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} cache: "pip" - cache-dependency-path: "./check_python_versions_requirements.txt" + cache-dependency-path: "./check_versions_requirements.txt" - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -63,25 +61,9 @@ jobs: ${{ env.X_PYTHON_MIN_VERSION }} \ ${{ env.X_PYTHON_MAX_VERSION }} - - name: Check Python Versions coincide with all pyproject.toml Files - run: | - for file in ../../sdk/pyproject.toml ../../compliance_tool/pyproject.toml ../../server/pyproject.toml; do - python check_python_versions_coincide.py \ - $file \ - ${{ env.X_PYTHON_MIN_VERSION }} \ - ${{ env.X_PYTHON_MAX_VERSION }} - done - - - name: Check Python Versions coincide with all Dockerfiles + - name: Check versions.yaml coincides with pyproject.toml and config files run: | - for file in ../../server/docker/repository/Dockerfile \ - ../../server/docker/registry/Dockerfile \ - ../../server/docker/discovery/Dockerfile; do - python check_python_versions_coincide.py --docker \ - $file \ - ${{ env.X_PYTHON_MIN_VERSION }} \ - ${{ env.X_PYTHON_MAX_VERSION }} - done + python check_global_versions_coincide.py sdk-test: # This job runs the unittests on the python versions specified down at the matrix @@ -91,12 +73,6 @@ jobs: python-version: ["3.10", "3.12"] env: COUCHDB_ADMIN_PASSWORD: "yo0Quai3" - # (2024-10-11, s-heppner) - # Specify the tag of the released schema files from https://github.com/admin-shell-io/aas-specs/releases - # that you want to use to test the serialization adapter against. - # Currently, we need to update this manually, however I'm afraid this is not possible to automatically infer, - # since it's heavily dependant of the version of the AAS specification we support. - AAS_SPECS_RELEASE_TAG: "v3.1.2" services: couchdb: image: couchdb:3 @@ -110,6 +86,7 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Verify Matrix Version matches Global Version run: | if [ "${{ matrix.python-version }}" != "${{ env.X_PYTHON_MIN_VERSION }}" ] && [ "${{ matrix.python-version }}" != "${{ env.X_PYTHON_MAX_VERSION }}" ]; then @@ -125,8 +102,8 @@ jobs: - name: Collect schema files from aas-specs run: | mkdir -p ./test/adapter/schema - curl -sSLf -o ./test/adapter/schema/aasJSONSchema.json https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/json/aas.json - curl -sSLf -o ./test/adapter/schema/aasXMLSchema.xsd https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/xml/AAS.xsd + curl -sSLf -o ./test/adapter/schema/aasJSONSchema.json https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.X_AAS_SPECS_RELEASE_TAG }}/schemas/json/aas.json + curl -sSLf -o ./test/adapter/schema/aasXMLSchema.xsd https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.X_AAS_SPECS_RELEASE_TAG }}/schemas/xml/AAS.xsd - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -150,6 +127,7 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -175,6 +153,7 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -206,6 +185,7 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MAX_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -228,6 +208,7 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -303,6 +284,7 @@ jobs: # Need tags so setuptools_scm builds the local sdk as >=1.0.0, else the loose # `basyx-python-sdk>=1.0.0` pin lets pip replace it with the PyPI release. fetch-depth: 0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -331,6 +313,7 @@ jobs: working-directory: ./compliance_tool steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -352,6 +335,7 @@ jobs: working-directory: ./server steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: @@ -382,6 +366,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions + - uses: ./.github/actions/build-server id: build with: diff --git a/compliance_tool/pyproject.toml b/compliance_tool/pyproject.toml index a26b85a5c..78261a86f 100644 --- a/compliance_tool/pyproject.toml +++ b/compliance_tool/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "mypy", + "mypy==1.15.0", "pycodestyle", "ruff==0.16.0", "codeblocks", diff --git a/etc/scripts/check_global_versions_coincide.py b/etc/scripts/check_global_versions_coincide.py new file mode 100644 index 000000000..e5929ad75 --- /dev/null +++ b/etc/scripts/check_global_versions_coincide.py @@ -0,0 +1,183 @@ +""" +This script dynamically verifies that the modules respect the versions defined in the global versions.yaml. +The python version is handled separately (checked against `requires-python` in every pyproject and against the +`FROM python:X.Y` line of every listed Dockerfile). +The tools section is checked against all toml files. +The config section is checked against a config file containing the name in its path. +All other sections are ignored as they are intended for direct consumption elsewhere (mostly CI). +""" +import argparse +import re +import sys +from pathlib import Path +from typing import Any + +import tomli as tomllib +import yaml + + +def _load_yaml(path: Path) -> dict[str, Any]: + try: + return yaml.safe_load(path.read_text()) + except FileNotFoundError: + print(f"Error: `{path}` not found.", file=sys.stderr) + sys.exit(1) + + +def _load_toml(path: Path) -> dict[str, Any]: + try: + return tomllib.loads(path.read_text()) + except FileNotFoundError: + print(f"Error: `{path}` not found.", file=sys.stderr) + sys.exit(1) + + +def _dev_deps(pyproject: dict[str, Any]) -> list[str]: + return (pyproject.get("project", {}).get("optional-dependencies", {}).get("dev", [])) + + +def check_python(versions: dict[str, Any], pyproject_paths: list[Path], repo_root: Path) -> list[str]: + """Verify `python.min` matches `[project] requires-python = ">="` in every pyproject.""" + min_version = versions.get("python", {}).get("min") + if not min_version: + return ["versions.yaml: missing `python.min`"] + + expected = f">={min_version}" + errors: list[str] = [] + for path in pyproject_paths: + pyproject = _load_toml(path) + actual = pyproject.get("project", {}).get("requires-python") + display = path.relative_to(repo_root) + if actual is None: + errors.append(f"{display}: missing `project.requires-python`") + elif actual != expected: + errors.append(f"{display}: `requires-python` is `{actual}`, expected `{expected}`") + return errors + + +def check_dev_tools(versions: dict[str, Any], pyproject_paths: list[Path], repo_root: Path) -> list[str]: + """Verify each `dev_tools.: ` appears as `==` in every pyproject's dev deps.""" + tools: dict[str, str] = versions.get("dev_tools", {}) + errors: list[str] = [] + + for path in pyproject_paths: + pyproject = _load_toml(path) + deps = _dev_deps(pyproject) + display = path.relative_to(repo_root) + + for tool, version in tools.items(): + pattern = re.compile(rf"^{re.escape(tool)}\s*==\s*{re.escape(version)}$") + if not any(pattern.match(dep) for dep in deps): + errors.append(f"{display}: dev dependency `{tool}=={version}` missing " + f"from `[project.optional-dependencies].dev` (found: {deps})") + return errors + + +def check_dockerfiles(versions: dict[str, Any], dockerfile_paths: list[Path], repo_root: Path) -> list[str]: + """ + Verify the `FROM python:X.Y[-...]` line of every listed Dockerfile has an `X.Y` that falls + within [python.min, python.max]. + """ + min_str = versions.get("python", {}).get("min") + max_str = versions.get("python", {}).get("max") + if not min_str or not max_str: + return ["versions.yaml: `python.min` / `python.max` required for Dockerfile checks"] + + min_ver = tuple(int(p) for p in min_str.split(".")) + max_ver = tuple(int(p) for p in max_str.split(".")) + from_pattern = re.compile(r"^\s*FROM\s+python:(\d+)\.(\d+)", re.MULTILINE) + + errors: list[str] = [] + for path in dockerfile_paths: + try: + text = path.read_text() + except FileNotFoundError: + errors.append(f"{path.relative_to(repo_root)}: Dockerfile not found") + continue + + match = from_pattern.search(text) + display = path.relative_to(repo_root) + if not match: + errors.append(f"{display}: no `FROM python:X.Y` line found") + continue + + found = (int(match.group(1)), int(match.group(2))) + found_str = f"{found[0]}.{found[1]}" + if found < min_ver or found > max_ver: + errors.append(f"{display}: base image `python:{found_str}` outside supported " + f"range [{min_str}, {max_str}]") + return errors + + +def check_configs(versions: dict[str, Any], config_paths: list[Path], repo_root: Path) -> list[str]: + """ + Verify each `config..: ` appears in a listed config file whose path + contains ``. Config files are loaded as YAML. + """ + config_section: dict[str, dict[str, Any]] = versions.get("config", {}) + errors: list[str] = [] + + for module_name, expected_entries in config_section.items(): + matches = [p for p in config_paths if module_name in str(p)] + if not matches: + errors.append(f"config.{module_name}: no config file in `configuration_locations.configs` " + f"has `{module_name}` in its path") + continue + if len(matches) > 1: + errors.append(f"config.{module_name}: ambiguous — multiple config files match " + f"`{module_name}`: {[str(m.relative_to(repo_root)) for m in matches]}") + continue + + config_path = matches[0] + loaded = _load_yaml(config_path) + display = config_path.relative_to(repo_root) + + for key, expected in expected_entries.items(): + actual = loaded.get(key) if isinstance(loaded, dict) else None + if actual is None: + errors.append(f"{display}: missing entry `{key}` (expected `{expected!r}`)") + elif actual != expected: + errors.append(f"{display}: `{key}` is `{actual!r}`, expected `{expected!r}`") + return errors + + +def _resolve_paths(rel_paths: list[str], repo_root: Path) -> list[Path]: + return [(repo_root / p).resolve() for p in rel_paths] + + +def main(versions_yaml_path: Path) -> int: + versions = _load_yaml(versions_yaml_path) + repo_root = versions_yaml_path.resolve().parent + + locations = versions.get("configuration_locations", {}) + pyproject_paths = _resolve_paths(locations.get("pyprojects", []), repo_root) + config_paths = _resolve_paths(locations.get("configs", []), repo_root) + dockerfile_paths = _resolve_paths(locations.get("dockerfiles", []), repo_root) + + errors: list[str] = [] + errors += check_python(versions, pyproject_paths, repo_root) + errors += check_dev_tools(versions, pyproject_paths, repo_root) + errors += check_dockerfiles(versions, dockerfile_paths, repo_root) + errors += check_configs(versions, config_paths, repo_root) + + if errors: + print("Version drift:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + + print(f"Success: `{versions_yaml_path.name}` coincides with " + f"{len(pyproject_paths)} pyproject.toml file(s), " + f"{len(dockerfile_paths)} Dockerfile(s), and " + f"{len(config_paths)} config file(s).") + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--versions", + type=Path, + default=Path(__file__).resolve().parents[2] / "versions.yaml", + help="Path to versions.yaml.", ) + args = parser.parse_args() + sys.exit(main(args.versions)) diff --git a/etc/scripts/check_python_versions_coincide.py b/etc/scripts/check_python_versions_coincide.py deleted file mode 100644 index ecf0fdf3f..000000000 --- a/etc/scripts/check_python_versions_coincide.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -This helper script checks if the Python versions defined in a `pyproject.toml` or `Dockerfile` coincide with the given -`min_version` and `max_version` and returns an error if they don't. -""" -import argparse -import re -import sys - -from packaging.version import InvalidVersion, Version - - -def get_version_pyproject(file_path: str) -> str: - with open(file_path, "r") as f: - pyproject_content = f.read() - - match = re.search(r'requires-python\s*=\s*">=([\d.]+)"', pyproject_content) - if not match: - print(f"Error: `requires-python` field not found or invalid format in `{file_path}`") - sys.exit(1) - - return match.group(1) - -def get_version_dockerfile(file_path: str) -> str: - with open(file_path, "r") as f: - pyproject_content = f.read() - - match = re.search(r'^FROM\s+python:([\d.]+)', pyproject_content, re.MULTILINE) - if not match: - print(f"Error: Definition of base image `FROM python:x.x` not found in `{file_path}`") - sys.exit(1) - - return match.group(1) - -def main(file_path: str, is_dockerfile: bool, min_version: str, max_version: str) -> None: - # Load and check `requires-python` version from `pyproject.toml` - try: - if is_dockerfile: - used_version = get_version_dockerfile(file_path) - else: - used_version = get_version_pyproject(file_path) - - if Version(used_version) < Version(min_version): - print(f"Error: Python version in `{file_path}` ({used_version}) " - f"is smaller than `min_version` ({min_version}).") - sys.exit(1) - if Version(used_version) > Version(max_version): - print(f"Error: Python version in `{file_path}` ({used_version}) " - f"is greater than `max_version` ({max_version}).") - sys.exit(1) - - except FileNotFoundError: - print(f"Error: File not found: `{file_path}`.") - sys.exit(1) - - print(f"Success: Version in `{file_path}` ({used_version}) " - f"matches expected versions ([{min_version} to {max_version}]).") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Check Python version support and alignment with pyproject.toml or Dockerfile.") - parser.add_argument("file_path", help="Path to the `pyproject.toml` or `Dockerfile` file to check.") - parser.add_argument("--docker", action="store_true", - help="Set, if checking a `Dockerfile`, otherwise `pyproject.toml` is assumed.") - parser.add_argument("min_version", help="The minimum Python version.") - parser.add_argument("max_version", help="The maximum Python version.") - args = parser.parse_args() - - try: - main(args.file_path, args.docker, args.min_version, args.max_version) - except InvalidVersion: - print("Error: Invalid version format provided.") - sys.exit(1) diff --git a/etc/scripts/check_python_versions_requirements.txt b/etc/scripts/check_versions_requirements.txt similarity index 57% rename from etc/scripts/check_python_versions_requirements.txt rename to etc/scripts/check_versions_requirements.txt index a8597d494..908d9e570 100644 --- a/etc/scripts/check_python_versions_requirements.txt +++ b/etc/scripts/check_versions_requirements.txt @@ -1,2 +1,4 @@ requests>=2.23 packaging>=24.2 +PyYAML>=6.0 +tomli>=2.0 diff --git a/server/config.yaml b/server/config.yaml new file mode 100644 index 000000000..124c9bd37 --- /dev/null +++ b/server/config.yaml @@ -0,0 +1 @@ +api_base_path: "/api/v3.1" diff --git a/server/pyproject.toml b/server/pyproject.toml index e87990397..a3ed4b477 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "mypy", + "mypy==1.15.0", "pycodestyle", "ruff==0.16.0", "codeblocks", diff --git a/versions.yaml b/versions.yaml new file mode 100644 index 000000000..f91b876d4 --- /dev/null +++ b/versions.yaml @@ -0,0 +1,40 @@ +# This file contains constants that are used throughout the project and whose version might be changed. +# Python version, checked against module .tomls and consumed by CI pipeline +python: + min: "3.10" + max: "3.12" + +# Dependency versions (of those that are fixed globally). Checked against module .tomls dynamically. +dev_tools: + mypy: "1.15.0" + ruff: "0.16.0" + +# Spec versions, consumed by CI pipeline. +spec: + aas: "v3.1.2" + api_version: "v3.1" + +# This section lists entries that are expected in a certain modules config.yaml. +config: + server: # Name of the module + api_base_path: "/api/v3.1" # Entry that is expected to be present and of this particular value + + +# This section lists locations of toml and configuration files that are checked to coincide with the global versions +# above. +# /etc/scripts/check_global_versions_coincide.py dynamically checks against the files listed here +configuration_locations: + pyprojects: + - sdk/pyproject.toml + - server/pyproject.toml + - compliance_tool/pyproject.toml + + configs: + - server/config.yaml + + # Dockerfiles are checked for a `FROM python:X.Y[-...]` line; `X.Y` must fall + # within [python.min, python.max]. + dockerfiles: + - server/docker/repository/Dockerfile + - server/docker/registry/Dockerfile + - server/docker/discovery/Dockerfile From 292b5b95a5259f85b4e80d91ab2b6a906d8b56d0 Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Sun, 9 Aug 2026 17:37:32 +0200 Subject: [PATCH 2/7] .github/actions/load-version: Fix broken path --- .github/actions/load-versions/action.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index 164bfe29f..cd0fc4774 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -1,16 +1,8 @@ name: load-versions description: > - Loads values from the repo-root `versions.yaml` into environment variables - for the current job (via `$GITHUB_ENV`). Downstream steps consume via - `${{ env.X_* }}`. Requires `actions/checkout` to have run first. - - Environment variables set: - X_PYTHON_MIN_VERSION (python.min) - X_PYTHON_MAX_VERSION (python.max) - X_MYPY_VERSION (tools.mypy) - X_RUFF_VERSION (tools.ruff) - X_API_VERSION (spec.api_version) - X_AAS_SPECS_RELEASE_TAG (spec.aas) + Loads values from the repo-root versions.yaml into environment variables + for the current job (via GITHUB_ENV). Downstream steps consume the values + through the usual env context. Requires actions/checkout to have run first. runs: using: composite @@ -20,8 +12,8 @@ runs: { echo "X_PYTHON_MIN_VERSION=$(yq '.python.min' versions.yaml)" echo "X_PYTHON_MAX_VERSION=$(yq '.python.max' versions.yaml)" - echo "X_MYPY_VERSION=$(yq '.tools.mypy' versions.yaml)" - echo "X_RUFF_VERSION=$(yq '.tools.ruff' versions.yaml)" + echo "X_MYPY_VERSION=$(yq '.dev_tools.mypy' versions.yaml)" + echo "X_RUFF_VERSION=$(yq '.dev_tools.ruff' versions.yaml)" echo "X_API_VERSION=$(yq '.spec.api_version' versions.yaml)" echo "X_AAS_SPECS_RELEASE_TAG=$(yq '.spec.aas' versions.yaml)" } >> "$GITHUB_ENV" From 826b673634eb241e92788f486384068e1c479f51 Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Sun, 9 Aug 2026 17:38:58 +0200 Subject: [PATCH 3/7] .github/workflows/pr: Update outdated requirements file path --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 71f84a7e2..4666adcd7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -54,7 +54,7 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -r ./check_python_versions_requirements.txt + python -m pip install -r ./check_versions_requirements.txt - name: Check Supported Python Versions run: | python check_python_versions_supported.py \ From 4456d3f91a456470aa77339dd448fee5816d301f Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Mon, 10 Aug 2026 10:41:51 +0200 Subject: [PATCH 4/7] server: Update server to replace all occurences of base path --- server/README.md | 4 ++-- server/app/_config.py | 23 +++++++++++++++++++ server/{ => app}/config.yaml | 0 server/app/interfaces/discovery.py | 3 ++- server/app/interfaces/registry.py | 3 ++- server/app/interfaces/repository.py | 3 ++- server/docker/discovery/Dockerfile | 1 - server/docker/registry/Dockerfile | 1 - server/docker/repository/Dockerfile | 1 - server/pyproject.toml | 3 ++- .../test/interfaces/test_shells_asset_ids.py | 3 +-- versions.yaml | 2 +- 12 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 server/app/_config.py rename server/{ => app}/config.yaml (100%) diff --git a/server/README.md b/server/README.md index 1a3e42173..32fe120c7 100644 --- a/server/README.md +++ b/server/README.md @@ -61,11 +61,11 @@ The container can be configured via environment variables. The most important on | Variable | Description | Default | |-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| -| `API_BASE_PATH` | Base path under which the API is served. | `/api/v3.1/` | +| `API_BASE_PATH` | Base path under which the API is served. Overrides the default from `app/config.yaml`. | `/api/v3.1` | | `INPUT` | Path inside the container pointing to the directory from which the server takes its start-up data. The repository server takes *AASX*, *JSON* and *XML* files, while the registry server takes AAS/Submodel descriptors from *JSON* files only. | `/input` | | `STORAGE` | Path inside the container pointing to the directory used by the repository or registry server to persistently store data (*JSON*). | `/storage` | | `STORAGE_PERSISTENCY` | Flag to enable data persistence via the [LocalFileBackend][2]. AAS/Submodels (repository server) or AAS/Submodel descriptors (registry server) are stored as *JSON* files in the directory specified by `STORAGE`. Supplementary files, i.e. files referenced by `File` SubmodelElements, are not stored. If disabled, any changes made via the API are only stored in memory. | `False` | -| `STORAGE_OVERWRITE` | Flag to enable storage overwrite if `STORAGE_PERSISTENCY` is enabled. Any AAS/Submodel from the `INPUT` directory already present in the LocalFileBackend replaces its existing version. If disabled, the existing version is kept. | `False` | +| `STORAGE_OVERWRITE` | Flag to enable storage overwrite if `STORAGE_PERSISTENCY` is enabled. Any AAS/Submodel from the `INPUT` directory already present in the LocalFileBackend replaces its existing version. If disabled, the existing version is kept. | `False` | This implies the following start-up behaviour: diff --git a/server/app/_config.py b/server/app/_config.py new file mode 100644 index 000000000..6366c6031 --- /dev/null +++ b/server/app/_config.py @@ -0,0 +1,23 @@ +# Copyright (c) 2026 the Eclipse BaSyx Authors +# +# This program and the accompanying materials are made available under the terms of the MIT License, available in +# the LICENSE file of this project. +# +# SPDX-License-Identifier: MIT +""" +Reads config.yaml once and exposes the entries as module-level constants. +Consumers should import the constants directly rather than reading the file. +""" +from functools import cache +from importlib.resources import files +from typing import Any + +import yaml + + +@cache +def _cfg() -> dict[str, Any]: + return yaml.safe_load(files(__package__).joinpath("config.yaml").read_text()) + + +API_BASE_PATH: str = _cfg()["api_base_path"] diff --git a/server/config.yaml b/server/app/config.yaml similarity index 100% rename from server/config.yaml rename to server/app/config.yaml diff --git a/server/app/interfaces/discovery.py b/server/app/interfaces/discovery.py index 2a60eb3e9..5d97d8659 100644 --- a/server/app/interfaces/discovery.py +++ b/server/app/interfaces/discovery.py @@ -14,6 +14,7 @@ from werkzeug.wrappers import Request, Response from app import model as server_model +from app._config import API_BASE_PATH from app.adapter import jsonization from app.interfaces.base import APIResponse, BaseWSGIApp, HTTPApiDecoder from app.model import ServiceDescription, ServiceSpecificationProfileEnum @@ -119,7 +120,7 @@ def to_file(self, filename: str) -> None: class DiscoveryAPI(BaseWSGIApp): - def __init__(self, persistent_store: DiscoveryStore, base_path: str = "/api/v3.1"): + def __init__(self, persistent_store: DiscoveryStore, base_path: str = API_BASE_PATH): self.persistent_store: DiscoveryStore = persistent_store self.url_map = werkzeug.routing.Map( [ diff --git a/server/app/interfaces/registry.py b/server/app/interfaces/registry.py index f69e3a505..d48307bf3 100644 --- a/server/app/interfaces/registry.py +++ b/server/app/interfaces/registry.py @@ -16,6 +16,7 @@ from werkzeug.wrappers import Request, Response import app.model as server_model +from app._config import API_BASE_PATH from app.interfaces.base import APIResponse, HTTPApiDecoder, ObjectStoreWSGIApp, PagingMetadata, is_stripped_request from app.model import DictDescriptorStore, ServiceDescription, ServiceSpecificationProfileEnum from app.util.converters import IdentifierToBase64URLConverter, base64url_decode @@ -31,7 +32,7 @@ class RegistryAPI(ObjectStoreWSGIApp): - def __init__(self, object_store: model.AbstractObjectStore, base_path: str = "/api/v3.1"): + def __init__(self, object_store: model.AbstractObjectStore, base_path: str = API_BASE_PATH): self.object_store: model.AbstractObjectStore = object_store self.url_map = werkzeug.routing.Map( [ diff --git a/server/app/interfaces/repository.py b/server/app/interfaces/repository.py index 8f931c786..fc7b4b0a0 100644 --- a/server/app/interfaces/repository.py +++ b/server/app/interfaces/repository.py @@ -22,6 +22,7 @@ from werkzeug.exceptions import BadRequest, Conflict, NotFound from werkzeug.routing import MapAdapter, Rule, Submount +from app._config import API_BASE_PATH from app.interfaces.base import PagingMetadata from app.model import ServiceDescription, ServiceSpecificationProfileEnum from app.util.converters import IdentifierToBase64URLConverter, IdShortPathConverter, base64url_decode @@ -43,7 +44,7 @@ def __init__( self, object_store: model.AbstractObjectStore, file_store: aasx.AbstractSupplementaryFileContainer, - base_path: str = "/api/v3.1", + base_path: str = API_BASE_PATH, ): self.object_store: model.AbstractObjectStore = object_store self.file_store: aasx.AbstractSupplementaryFileContainer = file_store diff --git a/server/docker/discovery/Dockerfile b/server/docker/discovery/Dockerfile index 66c9618e8..a1b74b778 100644 --- a/server/docker/discovery/Dockerfile +++ b/server/docker/discovery/Dockerfile @@ -33,7 +33,6 @@ ENV NGINX_MAX_UPLOAD=1M ENV NGINX_WORKER_PROCESSES=1 ENV LISTEN_PORT=80 ENV CLIENT_BODY_BUFFER_SIZE=1M -ENV API_BASE_PATH=/api/v3.1/ # Copy the entrypoint that will generate Nginx additional configs COPY server/docker/common/entrypoint.sh /entrypoint.sh diff --git a/server/docker/registry/Dockerfile b/server/docker/registry/Dockerfile index df367f2d6..c4584b82f 100644 --- a/server/docker/registry/Dockerfile +++ b/server/docker/registry/Dockerfile @@ -34,7 +34,6 @@ ENV NGINX_MAX_UPLOAD=1M ENV NGINX_WORKER_PROCESSES=1 ENV LISTEN_PORT=80 ENV CLIENT_BODY_BUFFER_SIZE=1M -ENV API_BASE_PATH=/api/v3.1/ # Default values for the storage envs ENV INPUT=/input diff --git a/server/docker/repository/Dockerfile b/server/docker/repository/Dockerfile index bc58e3e65..14adc597b 100644 --- a/server/docker/repository/Dockerfile +++ b/server/docker/repository/Dockerfile @@ -35,7 +35,6 @@ ENV NGINX_MAX_UPLOAD=1M ENV NGINX_WORKER_PROCESSES=1 ENV LISTEN_PORT=80 ENV CLIENT_BODY_BUFFER_SIZE=1M -ENV API_BASE_PATH=/api/v3.1/ # Default values for the storage envs ENV INPUT=/input diff --git a/server/pyproject.toml b/server/pyproject.toml index a3ed4b477..f85f9ea38 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -38,6 +38,7 @@ requires-python = ">=3.10" dependencies = [ "urllib3>=1.26,<3", "Werkzeug>=3.0.3,<4", + "PyYAML>=6.0", ] [project.optional-dependencies] @@ -59,7 +60,7 @@ dev = [ packages = { find = { include = ["app*"], exclude = ["test*"] } } [tool.setuptools.package-data] -app = ["py.typed"] +app = ["py.typed", "config.yaml"] [tool.mypy] exclude = "build/" diff --git a/server/test/interfaces/test_shells_asset_ids.py b/server/test/interfaces/test_shells_asset_ids.py index e45af1b90..b534373db 100644 --- a/server/test/interfaces/test_shells_asset_ids.py +++ b/server/test/interfaces/test_shells_asset_ids.py @@ -9,14 +9,13 @@ import json import unittest +from app._config import API_BASE_PATH as BASE_PATH from app.interfaces.repository import WSGIApp from basyx.aas import model from basyx.aas.adapter.aasx import DictSupplementaryFileContainer from basyx.aas.examples.data.example_aas import create_full_example from werkzeug.test import Client -BASE_PATH = "/api/v3.1" - def _encode_asset_id(name: str, value: str) -> str: payload = json.dumps({"name": name, "value": value}) diff --git a/versions.yaml b/versions.yaml index f91b876d4..017497862 100644 --- a/versions.yaml +++ b/versions.yaml @@ -30,7 +30,7 @@ configuration_locations: - compliance_tool/pyproject.toml configs: - - server/config.yaml + - server/app/config.yaml # Dockerfiles are checked for a `FROM python:X.Y[-...]` line; `X.Y` must fall # within [python.min, python.max]. From 0c91456a6a95ea04db026063ce6caee969c83122 Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Mon, 10 Aug 2026 10:45:23 +0200 Subject: [PATCH 5/7] server/pyproject.toml: Add yaml types for mypy --- server/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/pyproject.toml b/server/pyproject.toml index f85f9ea38..84a741188 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "jsonschema~=4.7", "hypothesis~=6.13", "lxml-stubs~=0.5.1", + "types-PyYAML", ] [project.urls] From 01f6ede692030151b2f4cd5848379f9fc888e99b Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Mon, 10 Aug 2026 10:49:20 +0200 Subject: [PATCH 6/7] server/test: Remove now obsolete base path test --- server/test/test_api_base_path.py | 50 ------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 server/test/test_api_base_path.py diff --git a/server/test/test_api_base_path.py b/server/test/test_api_base_path.py deleted file mode 100644 index acf1ceed8..000000000 --- a/server/test/test_api_base_path.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2026 the Eclipse BaSyx Authors -# -# This program and the accompanying materials are made available under the terms of the MIT License, available in -# the LICENSE file of this project. -# -# SPDX-License-Identifier: MIT -import pathlib -import re -import unittest - -SERVER_ROOT = pathlib.Path(__file__).resolve().parent.parent - -API_PATH_REGEX = re.compile(r"/api/v[\d.]+") - -FILES_TO_CHECK = [ - # Server routes - SERVER_ROOT / "app" / "interfaces" / "discovery.py", - SERVER_ROOT / "app" / "interfaces" / "registry.py", - SERVER_ROOT / "app" / "interfaces" / "repository.py", - # Dockerfiles - SERVER_ROOT / "docker" / "discovery" / "Dockerfile", - SERVER_ROOT / "docker" / "registry" / "Dockerfile", - SERVER_ROOT / "docker" / "repository" / "Dockerfile", - # Tests - SERVER_ROOT / "test" / "interfaces" / "test_shells_asset_ids.py", -] - - -def _extract(path: pathlib.Path) -> str: - match = API_PATH_REGEX.search(path.read_text()) - if match is None: - raise AssertionError(str(path.relative_to(SERVER_ROOT)) + ": no base path found") - return match.group(0) - - -def _list_divergences(values) -> str: - output = "API base path diverges across files:\n" - for p, v in values.items(): - output += f"\n{p} {v}" - return output - - -class APIBasePathConsistencyTest(unittest.TestCase): - def test_base_path_aligned_across_known_files(self) -> None: - values = {} - for path in FILES_TO_CHECK: - values[str(path.relative_to(SERVER_ROOT))] = _extract(path) - - distinct = set(values.values()) - self.assertEqual(1, len(distinct), _list_divergences(values)) From 7192d207fa8f17afe1a35bdebcff3291a85facef Mon Sep 17 00:00:00 2001 From: Joshua Benning Date: Wed, 12 Aug 2026 12:12:20 +0200 Subject: [PATCH 7/7] Bump python version and change to toml configs --- .github/actions/load-versions/action.yml | 14 ++-- .github/workflows/pr.yml | 4 +- .github/workflows/release.yml | 8 +- compliance_tool/pyproject.toml | 2 +- etc/scripts/check_global_versions_coincide.py | 80 ++++++++++--------- etc/scripts/check_versions_requirements.txt | 2 - ruff.toml | 2 +- sdk/.readthedocs.yaml | 2 +- sdk/pyproject.toml | 2 +- server/app/_config.py | 7 +- server/app/config.toml | 1 + server/app/config.yaml | 1 - server/pyproject.toml | 6 +- versions.toml | 38 +++++++++ versions.yaml | 40 ---------- 15 files changed, 102 insertions(+), 107 deletions(-) create mode 100644 server/app/config.toml delete mode 100644 server/app/config.yaml create mode 100644 versions.toml delete mode 100644 versions.yaml diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index cd0fc4774..0d55fb038 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -1,6 +1,6 @@ name: load-versions description: > - Loads values from the repo-root versions.yaml into environment variables + Loads values from the repo-root versions.toml into environment variables for the current job (via GITHUB_ENV). Downstream steps consume the values through the usual env context. Requires actions/checkout to have run first. @@ -10,10 +10,10 @@ runs: - shell: bash run: | { - echo "X_PYTHON_MIN_VERSION=$(yq '.python.min' versions.yaml)" - echo "X_PYTHON_MAX_VERSION=$(yq '.python.max' versions.yaml)" - echo "X_MYPY_VERSION=$(yq '.dev_tools.mypy' versions.yaml)" - echo "X_RUFF_VERSION=$(yq '.dev_tools.ruff' versions.yaml)" - echo "X_API_VERSION=$(yq '.spec.api_version' versions.yaml)" - echo "X_AAS_SPECS_RELEASE_TAG=$(yq '.spec.aas' versions.yaml)" + echo "X_PYTHON_MIN_VERSION=$(yq -p toml '.python.min' versions.toml)" + echo "X_PYTHON_MAX_VERSION=$(yq -p toml '.python.max' versions.toml)" + echo "X_MYPY_VERSION=$(yq -p toml '.dev_tools.mypy' versions.toml)" + echo "X_RUFF_VERSION=$(yq -p toml '.dev_tools.ruff' versions.toml)" + echo "X_API_VERSION=$(yq -p toml '.spec.api_version' versions.toml)" + echo "X_AAS_SPECS_RELEASE_TAG=$(yq -p toml '.spec.aas' versions.toml)" } >> "$GITHUB_ENV" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4666adcd7..86937a5f2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.12"] + python-version: ["3.11", "3.12"] env: COUCHDB_ADMIN_PASSWORD: "yo0Quai3" services: @@ -238,7 +238,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.12"] + python-version: ["3.11", "3.12"] defaults: run: working-directory: ./compliance_tool diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 36c0a04ba..1a3bf5b18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,10 +19,10 @@ jobs: working-directory: ./sdk steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -46,10 +46,10 @@ jobs: working-directory: ./compliance_tool steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/compliance_tool/pyproject.toml b/compliance_tool/pyproject.toml index 78261a86f..8da0e3da6 100644 --- a/compliance_tool/pyproject.toml +++ b/compliance_tool/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable" ] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "pyecma376-2>=0.2.4", "jsonschema>=4.21.1", diff --git a/etc/scripts/check_global_versions_coincide.py b/etc/scripts/check_global_versions_coincide.py index e5929ad75..8c13a394b 100644 --- a/etc/scripts/check_global_versions_coincide.py +++ b/etc/scripts/check_global_versions_coincide.py @@ -1,5 +1,5 @@ """ -This script dynamically verifies that the modules respect the versions defined in the global versions.yaml. +This script dynamically verifies that the modules respect the versions defined in the global versions.toml. The python version is handled separately (checked against `requires-python` in every pyproject and against the `FROM python:X.Y` line of every listed Dockerfile). The tools section is checked against all toml files. @@ -9,20 +9,10 @@ import argparse import re import sys +import tomllib from pathlib import Path from typing import Any -import tomli as tomllib -import yaml - - -def _load_yaml(path: Path) -> dict[str, Any]: - try: - return yaml.safe_load(path.read_text()) - except FileNotFoundError: - print(f"Error: `{path}` not found.", file=sys.stderr) - sys.exit(1) - def _load_toml(path: Path) -> dict[str, Any]: try: @@ -33,14 +23,14 @@ def _load_toml(path: Path) -> dict[str, Any]: def _dev_deps(pyproject: dict[str, Any]) -> list[str]: - return (pyproject.get("project", {}).get("optional-dependencies", {}).get("dev", [])) + return pyproject.get("project", {}).get("optional-dependencies", {}).get("dev", []) def check_python(versions: dict[str, Any], pyproject_paths: list[Path], repo_root: Path) -> list[str]: """Verify `python.min` matches `[project] requires-python = ">="` in every pyproject.""" min_version = versions.get("python", {}).get("min") if not min_version: - return ["versions.yaml: missing `python.min`"] + return ["versions.toml: missing `python.min`"] expected = f">={min_version}" errors: list[str] = [] @@ -56,7 +46,7 @@ def check_python(versions: dict[str, Any], pyproject_paths: list[Path], repo_roo def check_dev_tools(versions: dict[str, Any], pyproject_paths: list[Path], repo_root: Path) -> list[str]: - """Verify each `dev_tools.: ` appears as `==` in every pyproject's dev deps.""" + """Verify each `dev_tools. = ` appears as `==` in every pyproject's dev deps.""" tools: dict[str, str] = versions.get("dev_tools", {}) errors: list[str] = [] @@ -68,8 +58,10 @@ def check_dev_tools(versions: dict[str, Any], pyproject_paths: list[Path], repo_ for tool, version in tools.items(): pattern = re.compile(rf"^{re.escape(tool)}\s*==\s*{re.escape(version)}$") if not any(pattern.match(dep) for dep in deps): - errors.append(f"{display}: dev dependency `{tool}=={version}` missing " - f"from `[project.optional-dependencies].dev` (found: {deps})") + errors.append( + f"{display}: dev dependency `{tool}=={version}` missing " + f"from `[project.optional-dependencies].dev` (found: {deps})" + ) return errors @@ -81,7 +73,7 @@ def check_dockerfiles(versions: dict[str, Any], dockerfile_paths: list[Path], re min_str = versions.get("python", {}).get("min") max_str = versions.get("python", {}).get("max") if not min_str or not max_str: - return ["versions.yaml: `python.min` / `python.max` required for Dockerfile checks"] + return ["versions.toml: `python.min` / `python.max` required for Dockerfile checks"] min_ver = tuple(int(p) for p in min_str.split(".")) max_ver = tuple(int(p) for p in max_str.split(".")) @@ -104,15 +96,17 @@ def check_dockerfiles(versions: dict[str, Any], dockerfile_paths: list[Path], re found = (int(match.group(1)), int(match.group(2))) found_str = f"{found[0]}.{found[1]}" if found < min_ver or found > max_ver: - errors.append(f"{display}: base image `python:{found_str}` outside supported " - f"range [{min_str}, {max_str}]") + errors.append( + f"{display}: base image `python:{found_str}` outside supported " + f"range [{min_str}, {max_str}]" + ) return errors def check_configs(versions: dict[str, Any], config_paths: list[Path], repo_root: Path) -> list[str]: """ - Verify each `config..: ` appears in a listed config file whose path - contains ``. Config files are loaded as YAML. + Verify each `config.. = ` appears in a listed config file whose path + contains ``. Config files are loaded as TOML. """ config_section: dict[str, dict[str, Any]] = versions.get("config", {}) errors: list[str] = [] @@ -120,20 +114,24 @@ def check_configs(versions: dict[str, Any], config_paths: list[Path], repo_root: for module_name, expected_entries in config_section.items(): matches = [p for p in config_paths if module_name in str(p)] if not matches: - errors.append(f"config.{module_name}: no config file in `configuration_locations.configs` " - f"has `{module_name}` in its path") + errors.append( + f"config.{module_name}: no config file in `configuration_locations.configs` " + f"has `{module_name}` in its path" + ) continue if len(matches) > 1: - errors.append(f"config.{module_name}: ambiguous — multiple config files match " - f"`{module_name}`: {[str(m.relative_to(repo_root)) for m in matches]}") + errors.append( + f"config.{module_name}: ambiguous — multiple config files match " + f"`{module_name}`: {[str(m.relative_to(repo_root)) for m in matches]}" + ) continue config_path = matches[0] - loaded = _load_yaml(config_path) + loaded = _load_toml(config_path) display = config_path.relative_to(repo_root) for key, expected in expected_entries.items(): - actual = loaded.get(key) if isinstance(loaded, dict) else None + actual = loaded.get(key) if actual is None: errors.append(f"{display}: missing entry `{key}` (expected `{expected!r}`)") elif actual != expected: @@ -145,9 +143,9 @@ def _resolve_paths(rel_paths: list[str], repo_root: Path) -> list[Path]: return [(repo_root / p).resolve() for p in rel_paths] -def main(versions_yaml_path: Path) -> int: - versions = _load_yaml(versions_yaml_path) - repo_root = versions_yaml_path.resolve().parent +def main(versions_toml_path: Path) -> int: + versions = _load_toml(versions_toml_path) + repo_root = versions_toml_path.resolve().parent locations = versions.get("configuration_locations", {}) pyproject_paths = _resolve_paths(locations.get("pyprojects", []), repo_root) @@ -166,18 +164,22 @@ def main(versions_yaml_path: Path) -> int: print(f" - {e}", file=sys.stderr) return 1 - print(f"Success: `{versions_yaml_path.name}` coincides with " - f"{len(pyproject_paths)} pyproject.toml file(s), " - f"{len(dockerfile_paths)} Dockerfile(s), and " - f"{len(config_paths)} config file(s).") + print( + f"Success: `{versions_toml_path.name}` coincides with " + f"{len(pyproject_paths)} pyproject.toml file(s), " + f"{len(dockerfile_paths)} Dockerfile(s), and " + f"{len(config_paths)} config file(s)." + ) return 0 if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--versions", - type=Path, - default=Path(__file__).resolve().parents[2] / "versions.yaml", - help="Path to versions.yaml.", ) + parser.add_argument( + "--versions", + type=Path, + default=Path(__file__).resolve().parents[2] / "versions.toml", + help="Path to versions.toml.", + ) args = parser.parse_args() sys.exit(main(args.versions)) diff --git a/etc/scripts/check_versions_requirements.txt b/etc/scripts/check_versions_requirements.txt index 908d9e570..a8597d494 100644 --- a/etc/scripts/check_versions_requirements.txt +++ b/etc/scripts/check_versions_requirements.txt @@ -1,4 +1,2 @@ requests>=2.23 packaging>=24.2 -PyYAML>=6.0 -tomli>=2.0 diff --git a/ruff.toml b/ruff.toml index f62d1b261..2ff2f43ba 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,5 @@ line-length = 120 # matches the current pycodestyle --max-line-length 120 -target-version = "py310" # matches X_PYTHON_MIN_VERSION in pr.yml +target-version = "py311" [format] quote-style = "double" diff --git a/sdk/.readthedocs.yaml b/sdk/.readthedocs.yaml index 7fd098a4c..98512b12b 100644 --- a/sdk/.readthedocs.yaml +++ b/sdk/.readthedocs.yaml @@ -6,7 +6,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.10" + python: "3.11" sphinx: configuration: docs/source/conf.py diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index ba010aa4c..5627475a2 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable" ] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "lxml>=6.0.2", "python-dateutil>=2.8,<3", diff --git a/server/app/_config.py b/server/app/_config.py index 6366c6031..2a7482572 100644 --- a/server/app/_config.py +++ b/server/app/_config.py @@ -5,19 +5,18 @@ # # SPDX-License-Identifier: MIT """ -Reads config.yaml once and exposes the entries as module-level constants. +Reads config.toml once and exposes the entries as module-level constants. Consumers should import the constants directly rather than reading the file. """ +import tomllib from functools import cache from importlib.resources import files from typing import Any -import yaml - @cache def _cfg() -> dict[str, Any]: - return yaml.safe_load(files(__package__).joinpath("config.yaml").read_text()) + return tomllib.loads(files(__package__).joinpath("config.toml").read_text()) API_BASE_PATH: str = _cfg()["api_base_path"] diff --git a/server/app/config.toml b/server/app/config.toml new file mode 100644 index 000000000..2aa241e22 --- /dev/null +++ b/server/app/config.toml @@ -0,0 +1 @@ +api_base_path = "/api/v3.1" diff --git a/server/app/config.yaml b/server/app/config.yaml deleted file mode 100644 index 124c9bd37..000000000 --- a/server/app/config.yaml +++ /dev/null @@ -1 +0,0 @@ -api_base_path: "/api/v3.1" diff --git a/server/pyproject.toml b/server/pyproject.toml index 84a741188..4b8f1e1b3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -34,11 +34,10 @@ classifiers = [ "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable" ] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "urllib3>=1.26,<3", "Werkzeug>=3.0.3,<4", - "PyYAML>=6.0", ] [project.optional-dependencies] @@ -51,7 +50,6 @@ dev = [ "jsonschema~=4.7", "hypothesis~=6.13", "lxml-stubs~=0.5.1", - "types-PyYAML", ] [project.urls] @@ -61,7 +59,7 @@ dev = [ packages = { find = { include = ["app*"], exclude = ["test*"] } } [tool.setuptools.package-data] -app = ["py.typed", "config.yaml"] +app = ["py.typed", "config.toml"] [tool.mypy] exclude = "build/" diff --git a/versions.toml b/versions.toml new file mode 100644 index 000000000..9131a474b --- /dev/null +++ b/versions.toml @@ -0,0 +1,38 @@ +[python] +min = "3.11" +max = "3.12" + +# Dependency versions (of those that are fixed globally). Checked against module .tomls dynamically. +[dev_tools] +mypy = "1.15.0" +ruff = "0.16.0" + +# Spec versions, consumed by CI pipeline. +[spec] +aas = "v3.1.2" +api_version = "v3.1" + +# This section lists entries that are expected in a certain modules config.yaml +[config.server] +api_base_path = "/api/v3.1" + +# This section lists locations of toml and configuration files that are checked to coincide with the global versions +# above. +[configuration_locations] +pyprojects = [ + "sdk/pyproject.toml", + "server/pyproject.toml", + "compliance_tool/pyproject.toml", +] + +configs = [ + "server/app/config.toml", +] + +# Dockerfiles are checked for a `FROM python:X.Y[-...]` line; `X.Y` must fall +# within [python.min, python.max]. +dockerfiles = [ + "server/docker/repository/Dockerfile", + "server/docker/registry/Dockerfile", + "server/docker/discovery/Dockerfile", +] diff --git a/versions.yaml b/versions.yaml deleted file mode 100644 index 017497862..000000000 --- a/versions.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# This file contains constants that are used throughout the project and whose version might be changed. -# Python version, checked against module .tomls and consumed by CI pipeline -python: - min: "3.10" - max: "3.12" - -# Dependency versions (of those that are fixed globally). Checked against module .tomls dynamically. -dev_tools: - mypy: "1.15.0" - ruff: "0.16.0" - -# Spec versions, consumed by CI pipeline. -spec: - aas: "v3.1.2" - api_version: "v3.1" - -# This section lists entries that are expected in a certain modules config.yaml. -config: - server: # Name of the module - api_base_path: "/api/v3.1" # Entry that is expected to be present and of this particular value - - -# This section lists locations of toml and configuration files that are checked to coincide with the global versions -# above. -# /etc/scripts/check_global_versions_coincide.py dynamically checks against the files listed here -configuration_locations: - pyprojects: - - sdk/pyproject.toml - - server/pyproject.toml - - compliance_tool/pyproject.toml - - configs: - - server/app/config.yaml - - # Dockerfiles are checked for a `FROM python:X.Y[-...]` line; `X.Y` must fall - # within [python.min, python.max]. - dockerfiles: - - server/docker/repository/Dockerfile - - server/docker/registry/Dockerfile - - server/docker/discovery/Dockerfile