imap_processing/.github/workflows/test.yml at dev · IMAP-Science-Operations-Center/imap_processing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
name: Tests

on:
  push:
    branches:
      - dev
  pull_request:
  workflow_dispatch:

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, ubuntu-latest, macos-latest]
        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
    env:
      # External test data is only downloaded (and cached) on this one runner
      USE_EXTERNAL_DATA: ${{ contains(matrix.os, 'ubuntu') && matrix.python-version == '3.10' }}
    defaults:
      run:
        shell: bash


    steps:
      - uses: actions/checkout@v6
        with:
          # We need the full history to generate the proper version number
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - uses: Gr1N/setup-poetry@v9
        with:
          poetry-version: "2.3.4"


      - name: Install dependencies and app
        run: |
          # Need to install extra plugins first
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry install --extras "test"

      - name: Cache data directories
        uses: actions/cache@v6
        # We only download data on this runner, so only cache data from this run
        # so we don't get an empty cache on other runners overriding that data
        if: env.USE_EXTERNAL_DATA == 'true'
        with:
          path: |
            **/data/
            **/test_data/
          key: cdf-data-${{ hashFiles('**/data/*', '**/test_data/*') }}
          restore-keys: |
            cdf-data-

      - name: Restore tracked files overwritten by the cache
        # A fallback restore via restore-keys can bring back stale copies of
        # git-tracked files under data/ and test_data/; reset them to the
        # checked-out versions while keeping the untracked downloaded data.
        if: env.USE_EXTERNAL_DATA == 'true'
        run: git checkout -- .

      - name: Testing
        id: test
        run: |
          # Ignore the network marks from the remote test environment
          if [ "$USE_EXTERNAL_DATA" = "true" ]; then
            # Download external data before we run our tests so that
            # we don't try to download them in parallel
            poetry run python imap_processing/tests/conftest.py
            poetry run pytest -vvv -n auto --color=yes --cov --cov-report=xml --log-disable=root
          else
            poetry run pytest -vvv -n auto --color=yes --cov --cov-report=xml  --log-disable=root -m "not external_kernel and not external_test_data"
          fi

      - name: Upload coverage reports to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  pandas-3-compatibility:
    name: Pandas 3 compatibility (Python ${{ matrix.python-version }})
    # This is an advisory compatibility check and must not block pull requests.
    continue-on-error: true
    runs-on: ubuntu-latest
    strategy:
      # Run both supported Python versions even if one compatibility job fails.
      fail-fast: false
      matrix:
        # The first Python version below is the oldest version with Pandas 3
        # support that is also supported by IMAP, while the second is the
        # newest Python version supported by IMAP.
        python-version: ['3.11', '3.14']
    defaults:
      run:
        shell: bash

    steps:
      - uses: actions/checkout@v4
        with:
          # We need the full history to generate the proper version number.
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - uses: Gr1N/setup-poetry@v9
        with:
          poetry-version: "2.3.4"

      - name: Install dependencies, app, and Pandas 3
        run: |
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry install --extras "test"
          poetry run python -m pip install "pandas>=3,<4"
          poetry run python -c "import pandas; print(f'Pandas {pandas.__version__}')"

      # Run tests without external data.
      - name: Test with Pandas 3
        run: |
          # Coverage is produced and uploaded by the blocking primary matrix. Do
          # not upload duplicate advisory results or affect the coverage report.
          poetry run pytest -vvv -n auto --color=yes --log-disable=root \
            -m "not external_kernel and not external_test_data"