Skip to content

ENH - GLOWS: Reduce scope of integration tests #235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leowerneck wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ENH - GLOWS: Reduce scope of integration tests #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions tests/integration/test_glows_processor_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import subprocess
import unittest
from datetime import timedelta, datetime
from functools import wraps
from functools import wraps, partial
from pathlib import Path
from typing import Callable
from unittest import skipIf
Expand All @@ -29,6 +29,10 @@
create_glows_l3a_from_dictionary,
)
from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT
from imap_l3_processing.glows.l3e.glows_l3e_initializer import (
identify_versions_for_l3e_output_files,
)
from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings
from imap_l3_processing.models import InputMetadata, VersionMap
from imap_l3_processing.utils import save_data
from tests.integration.integration_test_helpers import (
Expand Down Expand Up @@ -110,8 +114,9 @@ def decorated(self):
f"type=bind,src={l3_processing_dir}/run_local_input_data,dst=/run_local_input_data",
]

if imap_api_key := os.getenv("IMAP_API_KEY"):
args += ["-e", f"IMAP_API_KEY={imap_api_key}"]
# Pass by name so docker reads the value from the environment and the key isn't in error messages
if os.getenv("IMAP_API_KEY"):
args += ["-e", "IMAP_API_KEY"]

if imap_data_access_url := os.getenv("IMAP_DATA_ACCESS_URL"):
args += ["-e", f"IMAP_DATA_ACCESS_URL={imap_data_access_url}"]
Expand All @@ -123,6 +128,29 @@ def decorated(self):
return decorated


def only_process_l3e_for(repointings_to_keep: set[int]):
# Each repointing runs five survival probability executables and needs SPICE kernels covering it,
# so only select the repointings the test checks
def identify_subset(*args, **kwargs) -> GlowsL3eVersionsForRepointings:
def keep(versions: dict) -> dict:
return {n: v for n, v in versions.items() if n in repointings_to_keep}

r = identify_versions_for_l3e_output_files(*args, **kwargs)
return GlowsL3eVersionsForRepointings(
[n for n in r.repointing_numbers if n in repointings_to_keep],
keep(r.hi_90_repointings),
keep(r.hi_45_repointings),
keep(r.lo_repointings),
keep(r.ultra_sf_repointings),
keep(r.ultra_hf_repointings),
)

return patch(
"imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files",
side_effect=identify_subset,
)
Comment on lines +148 to +151

@jtniehof jtniehof Oct 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The implication I'm drawing is that this is now no longer quite as much of an end-to-end integration test because we're monkey-patching identify_versions_for_l3e_output_files and losing that link of the chain. Am I being a bit paranoid here? Do we separately check the expected set of repointings either here or in a separate unit test?



class TestGlowsProcessorIntegration(unittest.TestCase):
@with_tempdir
def test_glows_l3a(self, tmp_dir):
Expand Down Expand Up @@ -399,7 +427,8 @@ def test_l3bcde_first_time_processing(self):
)

processor = GlowsProcessor(processing_input, input_metadata)
processor.process()
with only_process_l3e_for({36, 49}):
processor.process()

expected_files = [
ScienceFilePath(
Expand Down Expand Up @@ -572,7 +601,8 @@ def test_l3bcde_automatic_reprocessing(self):
)

processor = GlowsProcessor(processing_input, input_metadata)
processor.process()
with only_process_l3e_for({36}):
processor.process()

expected_files = [
ScienceFilePath(
Expand Down Expand Up @@ -770,16 +800,24 @@ def test_run_glows_l3be_against_prod(self):
"dependency": json.loads(processing_input.serialize()),
"version": {
desc: {
"major_version": 1,
"major_version": 2,
"minor_version": 1,
}
for desc in descriptors_to_produce
},
}

with patch(
"imap_l3_data_processor._parse_cli_arguments"
) as mock_parse_cli_arguments:
# Nothing is uploaded, so reruns compute the same output names as earlier local runs; overwrite them
with (
patch(
"imap_l3_data_processor._parse_cli_arguments"
) as mock_parse_cli_arguments,
patch(
"imap_l3_processing.glows.glows_processor.save_data",
side_effect=partial(save_data, delete_if_present=True),
),
only_process_l3e_for({36}),
):
mock_arguments = Mock()
mock_arguments.instrument = "glows"
mock_arguments.data_level = "l3b"
Expand Down
12 changes: 6 additions & 6 deletions tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
ignore-after: 2026-08-13 12:39:07.208204+00:00
l3e_survival-probability-hi-45 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161
l3e_survival-probability-hi-90 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161
l3e_survival-probability-lo repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161
l3e_survival-probability-ul-hf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161
l3e_survival-probability-ul-sf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161
ignore-after: 2100-01-01 00:00:00+00:00
l3e_survival-probability-hi-45 repoint36

@jtniehof jtniehof Oct 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on the CS principle that the only numbers are zero, one, and many, does it make sense to include two repoints for at least one of the outputs?

l3e_survival-probability-hi-90 repoint36
l3e_survival-probability-lo repoint36
l3e_survival-probability-ul-hf repoint36
l3e_survival-probability-ul-sf repoint36
Loading