From 2df3c98c4cccab1795cfc5ca3cc59497e64c987f Mon Sep 17 00:00:00 2001 From: Adrian Altenhoff Date: Thu, 29 Oct 2026 22:32:10 +0100 Subject: [PATCH 1/2] Run the darwin reference tests in CI test_align compared pyopa with the darwin reference results, but was skipped as it takes about 20 minutes. Run a fixed subsample by default (every 20th result with an alignment, which also checks estimate_pam, and every 200th of the others; about 12 s), and all results if PYOPA_TEST_FULL is set. Failure messages give the row of the reference results. A new workflow runs the full comparison on pushes to master, weekly and on demand. --- .github/workflows/full-tests.yml | 37 ++++++++++++++++++++++++++++ test/test_darwin.py | 42 +++++++++++++++++--------------- 2 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/full-tests.yml diff --git a/.github/workflows/full-tests.yml b/.github/workflows/full-tests.yml new file mode 100644 index 0000000..7de5040 --- /dev/null +++ b/.github/workflows/full-tests.yml @@ -0,0 +1,37 @@ +name: Full darwin tests + +# compares with all darwin reference results, which takes too long for every +# pull request (the regular tests use a subsample) +on: + push: + branches: [ master ] + schedule: + - cron: "0 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + full-tests: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + submodules: true + - name: Set up uv and Python + uses: astral-sh/setup-uv@v10.3.2 + with: + python-version: "3.14" + activate-environment: true + - name: Installation + env: + PYOPA_REQUIRE_OPENMP: "1" + run: | + uv pip install -v . + uv pip install --group test + - name: Test with pytest + env: + PYOPA_TEST_FULL: "1" + run: pytest -v test/test_darwin.py diff --git a/test/test_darwin.py b/test/test_darwin.py index 4e37e74..2fd1879 100644 --- a/test/test_darwin.py +++ b/test/test_darwin.py @@ -98,21 +98,25 @@ def setUp(self): self.alignment_profiles[curr.s1_id] = p ''' - #def test_align(self): - # threading.stack_size(67104768) - # t = threading.Thread(None, self._align_t, 'Aligning Thread') - # t.start() - # t.join() - - #def _align_t(self): - @unittest.skip("skip long runningtest") + def sample_results(self): + """ + All darwin results if PYOPA_TEST_FULL is set (takes several minutes), otherwise a + fixed subsample: every 20th result with an alignment (which also checks + estimate_pam) and every 200th of the others. + """ + if os.environ.get('PYOPA_TEST_FULL'): + return list(enumerate(self.darwin_results)) + with_al = [(i, r) for i, r in enumerate(self.darwin_results) if r.als1 != ''] + without_al = [(i, r) for i, r in enumerate(self.darwin_results) if r.als1 == ''] + return sorted(with_al[::20] + without_al[::200], key=lambda x: x[0]) + def test_align(self): - print('Running alignment tests...') + results = self.sample_results() + print('Running alignment tests on %d of %d results...' % (len(results), len(self.darwin_results))) completed = 0 - max_alignments = len(self.darwin_results) - progress_step = int(np.ceil(max_alignments / 100)) + progress_step = int(np.ceil(len(results) / 100)) - for r in self.darwin_results: + for test_id, r in results: s1 = self.sequences[r.s1_id - 1] s2 = self.sequences[r.s2_id - 1] env = self.alignment_environments[r.matrix_nr - 1] @@ -138,24 +142,24 @@ def test_align(self): delta=r.ep_sim *10**(1-self.precision), msg='Incorrect EstimatePam similarity score: %.10f != %.10f.' 'Test id: %d' % - (ep_result[0], r.ep_sim, completed + 1)) + (ep_result[0], r.ep_sim, test_id + 1)) self.assertAlmostEqual(ep_result[1], r.ep_pamn, delta=r.ep_pamn*10**(1-self.precision)) self.assertAlmostEqual(ep_result[2], r.ep_var, delta=r.ep_var*10**(1-self.precision)) self.assertAlmostEqual(scalar_result_reference, r.score_double, places=self.precision, msg='Incorrect reference double score: %.8f. The correct score is: %.8f, test id: %d' - % (scalar_result_reference, r.score_double, completed + 1)) + % (scalar_result_reference, r.score_double, test_id + 1)) self.assertGreaterEqual(short_result, r.score_double, msg="Short score (%f) must be greater or equal" " than double score (%f), test id: %d'." - % (short_result, r.score_double, completed + 1) + % (short_result, r.score_double, test_id + 1) ) self.assertGreaterEqual(byte_result, r.score_double, msg="Byte score must be greater or equal than double score.") self.assertAlmostEqual(double_result, r.score_double, places=self.precision, msg='Incorrect double score: %.8f. The correct score is: %.8f, test id: %d' % - (double_result, r.score_double, completed + 1)) + (double_result, r.score_double, test_id + 1)) ''' if byte_result > r.threshold: self.assertGreaterEqual(byte_result, sys.float_info.max) @@ -163,7 +167,7 @@ def test_align(self): self.assertAlmostEqual(byte_result, r.score_byte, places=7, msg='Incorrect byte score: %.8f. The correct score is: %.8f, test id: %d' % - (byte_result, r.score_byte, completed + 1)) + (byte_result, r.score_byte, test_id + 1)) ''' if short_result > r.threshold: self.assertGreaterEqual(short_result, sys.float_info.max) @@ -171,13 +175,13 @@ def test_align(self): if short_result < r.score_short and (r.score_short - short_result) > 0.01: #print("Warning: python short score(%f) is less than darwin's, but still bigger" \ # " than the double score(%f) at id: %d!" \ - # % (short_result, r.score_double, completed + 1)) + # % (short_result, r.score_double, test_id + 1)) pass else: self.assertAlmostEqual(short_result, r.score_short, places=self.precision, msg='Incorrect short score: %.8f. The correct score is: %.8f, test id: %d' % - (short_result, r.score_short, completed + 1)) + (short_result, r.score_short, test_id + 1)) completed += 1 if (completed % progress_step == 0): From 9538a94599c269ac5a949019375f424dcf0796ec Mon Sep 17 00:00:00 2001 From: Adrian Altenhoff Date: Thu, 29 Oct 2026 22:58:26 +0100 Subject: [PATCH 2/2] Run the full darwin tests on pull requests changing computations --- .github/workflows/full-tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/full-tests.yml b/.github/workflows/full-tests.yml index 7de5040..06c6d58 100644 --- a/.github/workflows/full-tests.yml +++ b/.github/workflows/full-tests.yml @@ -1,10 +1,20 @@ name: Full darwin tests # compares with all darwin reference results, which takes too long for every -# pull request (the regular tests use a subsample) +# pull request (the regular tests use a subsample). Runs on pull requests that +# change the computations or the reference tests. on: push: branches: [ master ] + pull_request: + branches: [ master ] + paths: + - "c_source/**" + - "pyopa/backend/**" + - "setup.py" + - "test/test_darwin.py" + - "test/data/**" + - ".github/workflows/full-tests.yml" schedule: - cron: "0 3 * * 1" workflow_dispatch: