HyperTools 1.0: architecture refactor + bug hunt

.github/workflows/test.yml

@@ -2,9 +2,13 @@ name: Tests
 
 on:
   push:
-    branches: [ master, dev, dev-2.0 ]
+    branches: [ master, dev, dev-1.0, dev-1.0-refactor ]
   pull_request:
-    branches: [ master, dev ]
+    # must include the branches PRs actually TARGET, or the PR-target
+    # workflow never runs and green checks are only push-triggered ones
+    # (2026-07 release review, blocker #3: PR #272 targets dev-1.0, which
+    # was missing here). Fork PRs in particular only run via pull_request.
+    branches: [ master, dev, dev-1.0, dev-1.0-refactor ]
 
 jobs:
   test:
@@ -13,18 +17,22 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
+        # datawrangler 0.5.0+ supports pandas 3.0 (dw#30/#31) and py3.10-3.13. The pandas<3
+        # ceiling is lifted, so the resolver naturally picks pandas 3.x on py3.11-3.13 and
+        # pandas 2.x on py3.10 (pandas 3.0 requires py>=3.11) -- the matrix exercises both.
+        # A dedicated pinned-pandas-3 gate runs below on ubuntu/py3.12.
         python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 (pinned to SHA, release review #8)
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.4.0 (pinned to SHA)
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache pip dependencies
-      uses: actions/cache@v4
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.0 (pinned to SHA)
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
@@ -33,7 +41,7 @@ jobs:
           ${{ runner.os }}-pip-
 
     - name: Cache hypertools example datasets
-      uses: actions/cache@v4
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.0 (pinned to SHA)
       with:
         path: ~/hypertools_data
         key: hypertools-example-data-v1
@@ -45,7 +53,11 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
-        sudo apt-get install -y ffmpeg
+        # fonts-noto-cjk (GH #205): ubuntu-latest ships no CJK-covering font
+        # at all, so tests/test_multibyte.py's `requires_covering_font`
+        # skipif would silently skip every CJK-dependent test on this OS.
+        sudo apt-get install -y ffmpeg fonts-noto-cjk
+        fc-cache -f
 
     - name: Install system dependencies (macOS)
       if: matrix.os == 'macos-latest'
@@ -59,26 +71,92 @@ jobs:
       continue-on-error: true
 
     - name: Install package with dev dependencies
+      # GH #130 (hyp.io.lsl_stream, tests/test_lsl_streaming.py): pylsl is
+      # in the [dev] extra (pyproject.toml) with NO separate native liblsl
+      # install step on any of the three platforms below -- investigated
+      # during Task 13's implementation, pylsl>=1.16's wheels BUNDLE the
+      # native liblsl library for every platform this matrix runs on:
+      #   - ubuntu-latest (x86_64): manylinux_2_35_x86_64 wheel -> ships
+      #     pylsl/lib/liblsl.so (ubuntu-latest's glibc satisfies 2.35+)
+      #   - macos-latest (arm64/x86_64): macosx_11_0_universal2 wheel ->
+      #     ships pylsl/lib/liblsl.dylib; verified working end-to-end
+      #     locally (real in-process StreamOutlet -> StreamInlet round trip)
+      #   - windows-latest (x86_64): win_amd64 wheel -> ships
+      #     pylsl/lib/lsl.dll
+      # so all three CI platforms are "provisioned" and none is excluded;
+      # tests/test_lsl_streaming.py::test_ci_has_pylsl fails the build (not
+      # skip) if pylsl is ever unimportable here, e.g. if a future pylsl
+      # release drops the bundled binary for one of these wheel tags.
       run: |
         python -m pip install --upgrade pip
-        pip install -e ".[dev]"
+        # [torch] (also pulled in transitively by [dev]) is installed
+        # explicitly too so a future change to the dev extra can't
+        # silently drop torch coverage of the six autoencoder reducers
+        # (GH #162, tests/test_autoencoders.py). torch>=2.0 ships CPU
+        # wheels for all three CI platforms/Python versions here.
+        pip install -e ".[dev,torch]"
+
+    - name: Pre-fetch headless Chrome for kaleido (Plotly image/GIF export)
+      # tests/test_animation_export.py exports Plotly figures to GIF/MP4 via
+      # kaleido 1.x, which drives a headless Chrome. Pre-fetching it here (into
+      # choreographer's deps dir, where kaleido discovers it automatically)
+      # avoids a slow first-use Chrome download racing the test's own timeout
+      # on some runners (seen as an intermittent TimeoutError on
+      # windows/py3.13). Best effort: if the fetch fails, kaleido falls back to
+      # its own on-demand download, so this step never blocks the build.
+      run: plotly_get_chrome -y
+      continue-on-error: true
+
+    - name: Rebuild matplotlib font cache (Ubuntu only)
+      # GH #205: matplotlib scans installed fonts once and caches the
+      # result (~/.cache/matplotlib/fontlist-*.json). On ubuntu-latest
+      # runners that cache is built (by whatever process first imports
+      # matplotlib.font_manager -- here, `pip install`'s dependency
+      # resolution/build step, or a previous cached run) BEFORE
+      # fonts-noto-cjk is installed above, so `find_covering_font`
+      # (hypertools/plot/fonts.py) would scan a stale font list missing
+      # every Noto Sans CJK file and skip every CJK-dependent test in
+      # tests/test_multibyte.py. Force one rebuild here (expensive -- do
+      # it once per job, not from inside fonts.py on every call).
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        python -c "import matplotlib.font_manager as fm; fm._load_fontmanager(try_read_cache=False)"
 
     - name: Run pytest
       env:
         MPLBACKEND: Agg
+        # Authenticate the fivethirtyeight loader's GitHub API listing calls
+        # (tests/test_load_538_kaggle.py). The default GITHUB_TOKEN raises the
+        # REST API quota from 60 to 5000 requests/hour, so the ~12 concurrent
+        # matrix jobs sharing a runner IP pool don't exhaust the shared
+        # unauthenticated limit (403 rate-limit failures). See
+        # hypertools/io/sources.py::_github_api_headers.
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         pytest -v --tb=short
 
+    - name: Acceptance gate — pandas 3.0 (Ubuntu Python 3.12 only)
+      # Guarantees a pandas-3 run regardless of resolver choices (dw#30/#31 fix).
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
+      env:
+        MPLBACKEND: Agg
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        pip install "pandas>=3.0"
+        python -c "import pandas as pd; assert pd.__version__.split('.')[0] == '3', pd.__version__; print('pandas', pd.__version__)"
+        pytest -q
+
     - name: Run pytest with coverage (Ubuntu Python 3.12 only)
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
       env:
         MPLBACKEND: Agg
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         pytest --cov=hypertools --cov-report=xml --cov-report=term-missing
 
     - name: Upload coverage to Codecov (Ubuntu Python 3.12 only)
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 (pinned to SHA)
       with:
         file: ./coverage.xml
         flags: unittests
@@ -94,8 +172,149 @@ jobs:
 
     - name: Upload screenshots as artifacts (Ubuntu Python 3.12 only)
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0 (pinned to SHA)
       with:
         name: verification-screenshots
         path: tests/screenshots/
         retention-days: 30
+
+  # Release qualification: build the wheel + sdist ONCE, then install EACH
+  # artifact into its OWN fresh environment (not the editable source
+  # checkout) and smoke-test the public API + validate metadata. Both are
+  # exercised because wheel and sdist use different package-discovery/build
+  # paths -- a gap can exist in one and not the other (2026-07 release
+  # review, issues #8 wheel + follow-up sdist). Actions are SHA-pinned like
+  # the test job above.
+  wheel-smoke:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 (pinned to SHA)
+    - name: Set up Python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.4.0 (pinned to SHA)
+      with:
+        python-version: '3.12'
+    - name: Build wheel + sdist
+      run: |
+        python -m pip install --upgrade pip build twine
+        python -m build --outdir dist .
+        twine check dist/*
+    - name: Install the built WHEEL into a fresh venv and smoke-test
+      env:
+        MPLBACKEND: Agg
+      run: |
+        python -m venv /tmp/fresh-wheel
+        # install the wheel artifact itself -- NOT the source checkout
+        /tmp/fresh-wheel/bin/pip install --upgrade pip
+        /tmp/fresh-wheel/bin/pip install dist/*.whl
+        # run from /tmp (NOT the repo) so an accidental source-tree import
+        # can't mask a packaging gap; the script itself asserts it imported
+        # hypertools from site-packages
+        cd /tmp && /tmp/fresh-wheel/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke_test.py"
+    - name: Install the built SDIST into a fresh venv and smoke-test
+      env:
+        MPLBACKEND: Agg
+      run: |
+        python -m venv /tmp/fresh-sdist
+        # build the sdist FROM the sdist (its own build path), not the repo
+        /tmp/fresh-sdist/bin/pip install --upgrade pip
+        /tmp/fresh-sdist/bin/pip install dist/*.tar.gz
+        cd /tmp && /tmp/fresh-sdist/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke_test.py"
+
+  # Fresh-source docs build (2026-07 release review, blocker #2). Read the Docs
+  # no longer ships a generated docs/auto_examples/ tree, so every RTD build now
+  # executes ALL gallery examples and renders the Plotly snapshots via headless
+  # Chrome from scratch. This job reproduces exactly that contract: it builds
+  # from a pristine `git archive` (tracked files only), asserts the generated
+  # gallery is absent, provisions Chrome the same way RTD does, and runs Sphinx
+  # with -W -E -a (warnings are errors, no cached env, rebuild everything) so a
+  # missing browser or ANY gallery-execution failure fails the build HERE --
+  # before it reaches RTD. Mirrors .readthedocs.yaml (Python 3.11 + ffmpeg +
+  # plotly_get_chrome). Actions SHA-pinned like the jobs above.
+  docs-clean:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 (pinned to SHA)
+    - name: Set up Python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.4.0 (pinned to SHA)
+      with:
+        python-version: '3.11'
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        # ffmpeg: mp4 rendering of animated gallery examples (matches RTD).
+        # fonts-noto-cjk: CJK-covering font for the multibyte gallery text.
+        # pandoc: nbsphinx renders the hand-authored tutorials/*.ipynb via
+        # pandoc (Read the Docs' build image bundles pandoc; a bare GH runner
+        # does not, so declare it here or the notebook stage fails).
+        sudo apt-get install -y ffmpeg fonts-noto-cjk pandoc
+        fc-cache -f
+    - name: Export a pristine source tree (tracked files only)
+      run: |
+        mkdir -p /tmp/docs-clean
+        git archive --format=tar HEAD | tar -x -C /tmp/docs-clean
+        # the generated gallery must NOT be tracked: a fresh RTD build
+        # regenerates it, so its presence here would MASK execution failures
+        if [ -e /tmp/docs-clean/docs/auto_examples ]; then
+          echo "::error::docs/auto_examples is tracked; it must be generated fresh"
+          exit 1
+        fi
+    - name: Install package + docs requirements + headless Chrome
+      run: |
+        python -m pip install --upgrade pip
+        cd /tmp/docs-clean
+        pip install .
+        pip install -r docs/doc_requirements.txt
+        # headless Chrome for the Plotly gallery scraper (kaleido 1.x); RTD
+        # provisions it identically via .readthedocs.yaml's post_install job
+        plotly_get_chrome -y
+    - name: Build docs from clean source (warnings = errors, cold gallery)
+      env:
+        MPLBACKEND: Agg
+        # raise the fivethirtyeight/kaggle loader's GitHub API quota (some
+        # gallery examples load hosted datasets during execution)
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        cd /tmp/docs-clean/docs
+        python -m sphinx -b html -W -E -a . _build/html
+
+  # Dataset release gate (2026-07 review, findings #2 + #2-followup). Two
+  # things a green matrix run does NOT prove: (a) that the hosted-dataset
+  # checks actually ran -- the matrix SKIPS a dataset it can't download so
+  # transient outages don't block unrelated PRs; and (b) that the Dropbox/Drive
+  # URLs are still REACHABLE -- the loader accepts a hash-valid CACHE hit
+  # without contacting the host. This job forbids both gaps: it uses NO dataset
+  # cache and clears ~/hypertools_data first, so every artifact is fetched
+  # FRESH from its host, and HYPERTOOLS_REQUIRE_DATASETS=1 turns any
+  # download/load failure into a HARD failure (never a skip). So it genuinely
+  # proves host availability + integrity + baseline match, and reports the
+  # exact count. Actions SHA-pinned like the jobs above.
+  dataset-gate:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.2.2 (pinned to SHA)
+    - name: Set up Python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.4.0 (pinned to SHA)
+      with:
+        python-version: '3.12'
+    - name: Install package
+      run: |
+        python -m pip install --upgrade pip
+        # base install (NOT [dev]) so this gate also proves the hosted datasets
+        # load with only the base dependency stack -- plus pytest to run them
+        # and pytest-timeout to honor the suite's 20-min hung-download safety net
+        pip install . pytest pytest-timeout
+    - name: Require EVERY hosted dataset to DOWNLOAD FRESH + match the pinned baseline
+      env:
+        MPLBACKEND: Agg
+        HYPERTOOLS_REQUIRE_DATASETS: '1'
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        # NO cache above + clear any cache dir here => every dataset is fetched
+        # fresh from its host, so this is a genuine host-availability gate (not
+        # a cache-validity check). A download/load failure fails the job (it
+        # cannot skip); -s prints the exact count validated.
+        rm -rf ~/hypertools_data
+        python -m pytest -v -s \
+          tests/test_dataset_compat.py \
+          tests/test_dataset_integrity.py

.gitignore

@@ -22,3 +22,14 @@ tests/screenshots/
 docs/auto_examples/animation.mp4
 docs/auto_examples/searchindex.db
 docs/auto_examples/test-image.pdf
+
+# stray local virtualenvs / build outputs (audit X5-001/X5-006)
+hypertools-dev/
+dist/
+
+# audit working notes + evidence (moved to PR #272 comments; full trail on
+# the audit/release-1.0-2026-07 branch). Do not re-track (release review #10).
+notes/audit-1.0-2026-07/
+# generated sphinx-gallery output: regenerated at docs build time from
+# examples/*.py -- source examples stay tracked, generated gallery does not.
+docs/auto_examples/

.readthedocs.yaml

@@ -11,7 +11,26 @@ build:
   tools:
     python: "3.11"
     # You can also specify other tools here or in a separate tools block
+  # ffmpeg system binary: required by sphinx-gallery's
+  # matplotlib_animations = (True, 'mp4') setting in docs/conf.py to render
+  # the animated gallery examples as mp4 video
+  apt_packages:
+    - ffmpeg
+    # pandoc: nbsphinx renders the hand-authored tutorials/*.ipynb via pandoc.
+    # RTD's build image has historically bundled pandoc, but declare it
+    # explicitly so the notebook stage never depends on an implicit default
+    # (the docs-clean CI job installs it the same way).
+    - pandoc
   jobs:
+    post_install:
+      # Kaleido 1.x renders the gallery's Plotly examples (plot_interactive_backend,
+      # animate_plotly, ...) to static PNGs via HEADLESS CHROME -- the
+      # plotly_sg_scraper wired up in docs/conf.py. Read the Docs provisions no
+      # browser, so without this the two Plotly examples fail when Kaleido tries
+      # to launch Chrome and Sphinx exits non-zero (2026-07 release review,
+      # blocker #2). Fetch a pinned Chrome into choreographer's deps dir, where
+      # Kaleido discovers it automatically (no PATH wiring needed).
+      - plotly_get_chrome -y
     post_build:
       # Fix animated GIF thumbnails after documentation build
       - python docs/post_build.py