Modernize xapi-db-load: migrate to uv, pyproject.toml, PEP 735 dependency groups

.github/workflows/ci.yml

@@ -1,39 +1,41 @@
-name: Run all tests & checks
+name: CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+  workflow_call:
 
 jobs:
   run_tests:
-    name: tests
+    name: ${{ matrix.toxenv }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [ '3.12' ]
-    permissions:
-      # Gives the action the necessary permissions for publishing new
-      # comments in pull requests.
-      pull-requests: write
-      # Gives the action the necessary permissions for pushing data to the
-      # python-coverage-comment-action branch, and for editing existing
-      # comments (to avoid publishing multiple comments in the same PR)
-      contents: write
+        python-version: ["3.12"]
+        toxenv: [quality, docs, py312]
 
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: setup python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
-          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          python-version: "${{ matrix.python-version }}"
 
-      - name: Install pip
-        run: pip install -r requirements/pip.txt
+      - name: Install CI dependencies
+        run: uv sync --group ci
 
-      - name: Install Dependencies
-        run: pip install -r requirements/ci.txt
+      - name: Run tox
+        run: uv run tox -e ${{ matrix.toxenv }}
 
-      - name: Run Tests
-        run: tox
+      - name: Upload coverage to Codecov
+        if: matrix.toxenv == 'py312'
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unittests
+          fail_ci_if_error: true

MANIFEST.in

@@ -1,6 +1,10 @@
-include CHANGELOG.rst
-include LICENSE.txt
-include README.rst
-include requirements/base.in
-include requirements/constraints.txt
-recursive-include xapi-db-load *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.csv
+# Exclude development, test, and documentation folders
+prune .github
+prune docs
+prune tests
+
+# Exclude root level configuration and build files
+exclude Makefile
+exclude conftest.py
+exclude .gitignore
+exclude tox.ini

Makefile

@@ -29,40 +29,24 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	tox -e docs
 	$(BROWSER)docs/_build/html/index.html
 
-# Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
-PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
-
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -r requirements/pip-tools.txt
-	pip install -qr requirements/pip.txt
-	# Make sure to compile files after any other files they include!
-	$(PIP_COMPILE) --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in
-	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -qr requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
-	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
-	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
-	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
-	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
-	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
-	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+upgrade: ## update the uv.lock file with the latest packages satisfying pyproject.toml
+	uv run --with edx-lint edx_lint write_uv_constraints pyproject.toml
+	uv lock --upgrade
 
 quality: ## check coding style with pycodestyle and pylint
-	pylint xapi_db_load *.py
-	pycodestyle xapi_db_load  *.py
-	pydocstyle xapi_db_load *.py
+	pylint xapi_db_load
+	pycodestyle xapi_db_load
+	pydocstyle xapi_db_load
 	mypy xapi_db_load
-	isort --check-only --diff --recursive xapi_db_load *.py test_settings.py
-	python setup.py bdist_wheel
+	isort --check-only --diff xapi_db_load
+	uv run python -m build
 	twine check dist/*
 	make selfcheck
 
 
 requirements: ## install development environment requirements
-	pip install -r requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
-	pip-sync requirements/dev.txt requirements/private.*
+	uv sync --group dev
+	uv tool install tox --with tox-uv
 
 test: clean ## run tests in the current virtualenv
 	pytest

pyproject.toml

@@ -0,0 +1,163 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "xapi-db-load"
+version = "3.1.0"
+description = "Loads testing xAPI events into databases and LRSs"
+readme = "README.rst"
+requires-python = ">=3.12"
+license = "AGPL-3.0"
+license-files = ["LICENSE.txt"]
+authors = [
+    {name = "Open edX Project", email = "oscm@openedx.org"},
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["Python", "edx"]
+dependencies = [
+    "chdb",
+    "click",
+    "clickhouse-connect[async]",
+    "pyarrow",
+    "pyyaml",
+    "requests",
+    "smart_open[s3]",
+    "urwid",
+    "uvloop",
+]
+
+[project.scripts]
+xapi-db-load = "xapi_db_load.main:cli"
+
+[project.urls]
+Repository = "https://github.com/openedx/xapi-db-load"
+"Issue tracker" = "https://github.com/openedx/xapi-db-load/issues"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+exclude = ["tests*", "*.tests", "*.tests.*"]
+
+[tool.setuptools.package-data]
+"*" = [
+    "conf/**/*",
+    "public/**/*",
+    "static/**/*",
+    "templates/**/*",
+    "translations/**/*",
+]
+
+[dependency-groups]
+# Each group mirrors its requirements/*.in file exactly.
+# "-r other.in" in the .in file → {include-group = "other"} here.
+
+# From requirements/test.in
+test = [
+    {include-group = "base"},
+    "pytest-asyncio",
+    "pytest-cov",
+]
+
+# From requirements/quality.in
+quality = [
+    {include-group = "test"},
+    "edx-lint",
+    "isort",
+    "mypy",
+    "pycodestyle",
+    "pydocstyle",
+    "twine",
+]
+
+# From requirements/doc.in
+doc = [
+    {include-group = "test"},
+    "build",
+    "doc8",
+    "Sphinx",
+    "sphinx-book-theme",
+    "twine",
+]
+
+# From requirements/ci.in
+ci = [
+    "tox",
+    "tox-uv",
+]
+
+# From requirements/dev.in (without pip-tools since that's replaced by uv)
+dev = [
+    {include-group = "quality"},
+    {include-group = "ci"},
+    "diff-cover",
+]
+
+# base group (mirrors base.in — used via include-group from test)
+base = [
+    "chdb",
+    "click",
+    "clickhouse-connect[async]",
+    "pyarrow",
+    "pyyaml",
+    "requests",
+    "smart_open[s3]",
+    "urwid",
+    "uvloop",
+]
+
+[tool.edx_lint]
+uv_constraints = []
+
+[tool.uv]
+package = true
+
+#  DO NOT EDIT constraint-dependencies DIRECTLY.
+#  This list is managed by `edx_lint write_uv_constraints`
+#  and will be overwritten the next time `make upgrade` is run.
+#  - GLOBAL constraints: edit edx_lint/files/common_constraints.txt
+#  - REPO-SPECIFIC constraints: edit [tool.edx_lint].uv_constraints in this file
+constraint-dependencies = [
+    "Django<6.0",
+    "elasticsearch<7.14.0",
+]
+[tool.coverage.run]
+branch = true
+data_file = ".coverage"
+source = ["xapi_db_load"]
+omit = [
+    "tests",
+]
+
+[tool.coverage.report]
+show_missing = true
+
+[tool.pytest.ini_options]
+addopts = "--cov xapi_db_load --cov-report term-missing --cov-report xml --log-level=INFO"
+norecursedirs = [".* docs requirements site-packages"]
+
+[tool.isort]
+include_trailing_comma = true
+indent = "    "
+line_length = 120
+multi_line_output = 3
+skip = ["migrations"]
+
+[tool.mypy]
+python_version = "3.12"
+ignore_missing_imports = true
+check_untyped_defs = false
+warn_unused_ignores = true
+exclude = ["^docs/", "^build/", "^dist/"]
+
+[[tool.mypy.overrides]]
+module = "xapi_db_load.tests.*"
+ignore_errors = true