From a724487be9de96ff223a031ae32e42c382241af0 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 7 Oct 2026 17:15:57 -0400 Subject: [PATCH 1/3] Dockerfile_glows_integration.dockerignore: added to fix test failures --- Dockerfile_glows_integration.dockerignore | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Dockerfile_glows_integration.dockerignore diff --git a/Dockerfile_glows_integration.dockerignore b/Dockerfile_glows_integration.dockerignore new file mode 100644 index 00000000..59bd67b5 --- /dev/null +++ b/Dockerfile_glows_integration.dockerignore @@ -0,0 +1,18 @@ +# BuildKit uses this file instead of the root .dockerignore when building +# Dockerfile_glows_integration. The integration image needs tests, SPICE +# kernels, instrument team data and run_local.py in addition to the package. +* + +!pyproject.toml +!uv.lock +!imap_l3_data_processor.py +!imap_l3_processing/ +!scripts/ +!tests/ +!spice_kernels/ +!instrument_team_data/ +!run_local.py + +**/__pycache__/ +**/*.pyc +**/*.pyo From d877675629bce0de5b2b3fe48371e19557c6709f Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 7 Oct 2026 17:16:33 -0400 Subject: [PATCH 2/3] tests/integration/test_glows_processor_integration.py: ran ruff format & ruff check --fix --- .../test_glows_processor_integration.py | 627 ++++++++++++------ 1 file changed, 442 insertions(+), 185 deletions(-) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index 274ae639..e1f932d9 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -9,7 +9,7 @@ from functools import wraps from pathlib import Path from typing import Callable -from unittest import skipIf, skip +from unittest import skipIf from unittest.mock import patch, Mock import imap_data_access @@ -24,13 +24,25 @@ from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS from imap_l3_processing.glows.glows_processor import GlowsProcessor from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies -from imap_l3_processing.glows.l3a.utils import read_l2_glows_data, create_glows_l3a_from_dictionary +from imap_l3_processing.glows.l3a.utils import ( + read_l2_glows_data, + create_glows_l3a_from_dictionary, +) from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data -from tests.integration.integration_test_helpers import mock_imap_data_access, run_istp_compliance_check -from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path, \ - with_tempdir, get_run_local_data_path, run_periodically, create_mock_version_map +from tests.integration.integration_test_helpers import ( + mock_imap_data_access, + run_istp_compliance_check, +) +from tests.test_helpers import ( + get_test_data_path, + get_test_instrument_team_data_path, + with_tempdir, + get_run_local_data_path, + run_periodically, + create_mock_version_map, +) GLOWS_INTEGRATION_DATA_DIR = get_run_local_data_path( "glows_l3bcde_integration_data_dir" @@ -41,8 +53,12 @@ def generate_test_function_import_path(fn: Callable) -> str: path_to_imap_processing_dir = Path(tests.__file__).parent.parent - path_to_file = Path(fn.__code__.co_filename).relative_to(path_to_imap_processing_dir) - path_in_import_style = str(path_to_file).replace('.py', '.').replace(os.path.sep, '.') + path_to_file = Path(fn.__code__.co_filename).relative_to( + path_to_imap_processing_dir + ) + path_in_import_style = ( + str(path_to_file).replace(".py", ".").replace(os.path.sep, ".") + ) return path_in_import_style + fn.__qualname__ @@ -54,17 +70,34 @@ def decorated(self): else: l3_processing_dir = Path(tests.__file__).parent.parent - docker_build = subprocess.run(["docker", "build", "--platform", "linux/amd64", "-q", "-f", "Dockerfile_glows_integration", "."], - cwd=l3_processing_dir, capture_output=True) - image_hash = docker_build.stdout.strip().decode('utf-8') + docker_build = subprocess.run( + [ + "docker", + "build", + "--platform", + "linux/amd64", + "-q", + "-f", + "Dockerfile_glows_integration", + ".", + ], + cwd=l3_processing_dir, + capture_output=True, + ) + image_hash = docker_build.stdout.strip().decode("utf-8") print(f"Built docker container: {image_hash}") args = [ - "docker", "run", "--rm", - "--platform", "linux/amd64", - "--mount", f'type=bind,src={l3_processing_dir}/temp_cdf_data,dst=/temp_cdf_data', - "--mount", f'type=bind,src={l3_processing_dir}/run_local_input_data,dst=/run_local_input_data', + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "--mount", + f"type=bind,src={l3_processing_dir}/temp_cdf_data,dst=/temp_cdf_data", + "--mount", + f"type=bind,src={l3_processing_dir}/run_local_input_data,dst=/run_local_input_data", ] if imap_api_key := os.getenv("IMAP_API_KEY"): @@ -79,6 +112,7 @@ def decorated(self): return decorated + class TestGlowsProcessorIntegration(unittest.TestCase): @with_tempdir def test_glows_l3a(self, tmp_dir): @@ -89,35 +123,46 @@ def test_glows_l3a(self, tmp_dir): date_in_path = l2_science_file_path.start_date start_date = datetime.strptime(date_in_path, "%Y%m%d") end_date = start_date + timedelta(days=1) - l3a_descriptor = 'hist' + l3a_descriptor = "hist" input_metadata = InputMetadata( - instrument='glows', - data_level='l3a', + instrument="glows", + data_level="l3a", descriptor=l3a_descriptor, start_date=start_date, end_date=end_date, - version=create_mock_version_map(descriptor=l3a_descriptor,minor_version=1), - repointing=l2_science_file_path.repointing + version=create_mock_version_map(descriptor=l3a_descriptor, minor_version=1), + repointing=l2_science_file_path.repointing, ) expected_json_path = get_test_instrument_team_data_path( - "glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json") + "glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json" + ) with open(expected_json_path) as f: instrument_team_dict = json.load(f) - expected_output = create_glows_l3a_from_dictionary(instrument_team_dict, input_metadata) + expected_output = create_glows_l3a_from_dictionary( + instrument_team_dict, input_metadata + ) with CDF(str(input_l2_cdf_path)) as cdf_data: l2_glows_data = read_l2_glows_data(cdf_data) - dependencies = GlowsL3ADependencies(l2_glows_data, { - "calibration_data": get_test_instrument_team_data_path( - "glows/imap_glows_calibration-data_20100101_v002.dat"), - "settings": get_test_instrument_team_data_path("glows/imap_glows_pipeline-settings_20100101_v001.json"), - "time_dependent_bckgrd": get_test_instrument_team_data_path( - "glows/imap_glows_time-dep-bckgrd_20100101_v001.dat"), - "extra_heliospheric_bckgrd": get_test_instrument_team_data_path( - "glows/imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat"), - }) + dependencies = GlowsL3ADependencies( + l2_glows_data, + { + "calibration_data": get_test_instrument_team_data_path( + "glows/imap_glows_calibration-data_20100101_v002.dat" + ), + "settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings_20100101_v001.json" + ), + "time_dependent_bckgrd": get_test_instrument_team_data_path( + "glows/imap_glows_time-dep-bckgrd_20100101_v001.dat" + ), + "extra_heliospheric_bckgrd": get_test_instrument_team_data_path( + "glows/imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat" + ), + }, + ) processor = GlowsProcessor(ProcessingInputCollection(), input_metadata) l3a_data = processor.process_l3a(dependencies) @@ -130,53 +175,108 @@ def test_glows_l3a(self, tmp_dir): self.assertEqual(input_metadata.repointing, l3a_data.identifier) - np.testing.assert_allclose(actual_dict['photon_flux'], expected_dict['photon_flux'], rtol=1e-3) - np.testing.assert_allclose(actual_dict['photon_flux_uncertainty'], expected_dict['photon_flux_uncertainty'], - rtol=1e-3) - np.testing.assert_allclose(actual_dict['raw_histogram'], expected_dict['raw_histogram']) - np.testing.assert_allclose(actual_dict['exposure_times'], expected_dict['exposure_times'], rtol=1e-3) - np.testing.assert_allclose(actual_dict['number_of_bins'], expected_dict['number_of_bins']) - self.assertEqual(actual_dict['epoch'], expected_dict['epoch']) - np.testing.assert_allclose(actual_dict['epoch_delta'], expected_dict['epoch_delta']) - np.testing.assert_allclose(actual_dict['spin_angle'], expected_dict['spin_angle']) - np.testing.assert_allclose(actual_dict['spin_angle_delta'], expected_dict['spin_angle_delta']) - np.testing.assert_allclose(actual_dict['latitude'], expected_dict['latitude'], atol=1e-3) - np.testing.assert_allclose(actual_dict['longitude'], expected_dict['longitude'], atol=1e-3) - np.testing.assert_allclose(actual_dict['filter_temperature_average'], - expected_dict['filter_temperature_average']) - np.testing.assert_allclose(actual_dict['filter_temperature_std_dev'], - expected_dict['filter_temperature_std_dev']) - np.testing.assert_allclose(actual_dict['hv_voltage_average'], expected_dict['hv_voltage_average']) - np.testing.assert_allclose(actual_dict['hv_voltage_std_dev'], expected_dict['hv_voltage_std_dev']) - np.testing.assert_allclose(actual_dict['spin_period_average'], expected_dict['spin_period_average']) - np.testing.assert_allclose(actual_dict['spin_period_std_dev'], expected_dict['spin_period_std_dev']) - np.testing.assert_allclose(actual_dict['spin_period_ground_average'], - expected_dict['spin_period_ground_average']) - np.testing.assert_allclose(actual_dict['spin_period_ground_std_dev'], - expected_dict['spin_period_ground_std_dev']) - np.testing.assert_allclose(actual_dict['pulse_length_average'], expected_dict['pulse_length_average']) - np.testing.assert_allclose(actual_dict['pulse_length_std_dev'], expected_dict['pulse_length_std_dev']) - np.testing.assert_allclose(actual_dict['position_angle_offset_average'], - expected_dict['position_angle_offset_average']) - np.testing.assert_allclose(actual_dict['position_angle_offset_std_dev'], - expected_dict['position_angle_offset_std_dev']) - np.testing.assert_allclose(actual_dict['spin_axis_orientation_average'], - expected_dict['spin_axis_orientation_average']) - np.testing.assert_allclose(actual_dict['spin_axis_orientation_std_dev'], - expected_dict['spin_axis_orientation_std_dev']) - np.testing.assert_allclose(actual_dict['spacecraft_location_average'], - expected_dict['spacecraft_location_average']) - np.testing.assert_allclose(actual_dict['spacecraft_location_std_dev'], - expected_dict['spacecraft_location_std_dev']) - - self.assertEqual(actual_dict['input_metadata'], expected_dict['input_metadata']) + np.testing.assert_allclose( + actual_dict["photon_flux"], expected_dict["photon_flux"], rtol=1e-3 + ) + np.testing.assert_allclose( + actual_dict["photon_flux_uncertainty"], + expected_dict["photon_flux_uncertainty"], + rtol=1e-3, + ) + np.testing.assert_allclose( + actual_dict["raw_histogram"], expected_dict["raw_histogram"] + ) + np.testing.assert_allclose( + actual_dict["exposure_times"], expected_dict["exposure_times"], rtol=1e-3 + ) + np.testing.assert_allclose( + actual_dict["number_of_bins"], expected_dict["number_of_bins"] + ) + self.assertEqual(actual_dict["epoch"], expected_dict["epoch"]) + np.testing.assert_allclose( + actual_dict["epoch_delta"], expected_dict["epoch_delta"] + ) + np.testing.assert_allclose( + actual_dict["spin_angle"], expected_dict["spin_angle"] + ) + np.testing.assert_allclose( + actual_dict["spin_angle_delta"], expected_dict["spin_angle_delta"] + ) + np.testing.assert_allclose( + actual_dict["latitude"], expected_dict["latitude"], atol=1e-3 + ) + np.testing.assert_allclose( + actual_dict["longitude"], expected_dict["longitude"], atol=1e-3 + ) + np.testing.assert_allclose( + actual_dict["filter_temperature_average"], + expected_dict["filter_temperature_average"], + ) + np.testing.assert_allclose( + actual_dict["filter_temperature_std_dev"], + expected_dict["filter_temperature_std_dev"], + ) + np.testing.assert_allclose( + actual_dict["hv_voltage_average"], expected_dict["hv_voltage_average"] + ) + np.testing.assert_allclose( + actual_dict["hv_voltage_std_dev"], expected_dict["hv_voltage_std_dev"] + ) + np.testing.assert_allclose( + actual_dict["spin_period_average"], expected_dict["spin_period_average"] + ) + np.testing.assert_allclose( + actual_dict["spin_period_std_dev"], expected_dict["spin_period_std_dev"] + ) + np.testing.assert_allclose( + actual_dict["spin_period_ground_average"], + expected_dict["spin_period_ground_average"], + ) + np.testing.assert_allclose( + actual_dict["spin_period_ground_std_dev"], + expected_dict["spin_period_ground_std_dev"], + ) + np.testing.assert_allclose( + actual_dict["pulse_length_average"], expected_dict["pulse_length_average"] + ) + np.testing.assert_allclose( + actual_dict["pulse_length_std_dev"], expected_dict["pulse_length_std_dev"] + ) + np.testing.assert_allclose( + actual_dict["position_angle_offset_average"], + expected_dict["position_angle_offset_average"], + ) + np.testing.assert_allclose( + actual_dict["position_angle_offset_std_dev"], + expected_dict["position_angle_offset_std_dev"], + ) + np.testing.assert_allclose( + actual_dict["spin_axis_orientation_average"], + expected_dict["spin_axis_orientation_average"], + ) + np.testing.assert_allclose( + actual_dict["spin_axis_orientation_std_dev"], + expected_dict["spin_axis_orientation_std_dev"], + ) + np.testing.assert_allclose( + actual_dict["spacecraft_location_average"], + expected_dict["spacecraft_location_average"], + ) + np.testing.assert_allclose( + actual_dict["spacecraft_location_std_dev"], + expected_dict["spacecraft_location_std_dev"], + ) + + self.assertEqual(actual_dict["input_metadata"], expected_dict["input_metadata"]) istp_compliance_message = run_istp_compliance_check(output_cdf_path) print("ISTP Compliance:\n", istp_compliance_message) self.assertIn("PASSED variable checks", istp_compliance_message) def test_l3a_handles_l2_input_that_is_all_flagged(self): - bad_l2_cdf_path = get_test_data_path("glows/imap_glows_l2_hist_20251113-repoint00047_v003.cdf") + bad_l2_cdf_path = get_test_data_path( + "glows/imap_glows_l2_hist_20251113-repoint00047_v003.cdf" + ) l2_science_file_path = ScienceFilePath(bad_l2_cdf_path) @@ -255,74 +355,138 @@ def test_l3bcde_first_time_processing(self): INTEGRATION_TEST_DATA / "spice" / "imap_recon_20250415_20260415_v01.bsp", ] with mock_imap_data_access(GLOWS_INTEGRATION_DATA_DIR, input_files): - - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) for folder in ["data_l3b", "data_l3c", "data_l3d", "data_l3d_txt"]: path = PATH_TO_L3D_TOOLKIT / folder if path.exists(): shutil.rmtree(path) - processing_input = ProcessingInputCollection(RepointInput("imap_2026_090_01.repoint"), - AncillaryInput("imap_glows_force-reprocessing-config_20250101_v000.csv")) - - version_map = VersionMap({descriptor: Version(i, 2) for i, descriptor in enumerate(GLOWS_L3BCDE_DESCRIPTORS)}) - input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version=version_map, start_date=datetime(2000, 1, 1), - end_date=datetime(2000, 1, 1)) + processing_input = ProcessingInputCollection( + RepointInput("imap_2026_090_01.repoint"), + AncillaryInput( + "imap_glows_force-reprocessing-config_20250101_v000.csv" + ), + ) + + version_map = VersionMap( + { + descriptor: Version(i, 2) + for i, descriptor in enumerate(GLOWS_L3BCDE_DESCRIPTORS) + } + ) + input_metadata = InputMetadata( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version=version_map, + start_date=datetime(2000, 1, 1), + end_date=datetime(2000, 1, 1), + ) processor = GlowsProcessor(processing_input, input_metadata) processor.process() expected_files = [ - ScienceFilePath('imap_glows_l3b_ion-rate-profile_20251102-cr02304_v000.0001.cdf'), - ScienceFilePath('imap_glows_l3b_ion-rate-profile_20251130-cr02305_v000.0001.cdf'), - - ScienceFilePath('imap_glows_l3c_sw-profile_20251102-cr02304_v000.0001.cdf'), - ScienceFilePath('imap_glows_l3c_sw-profile_20251130-cr02305_v000.0001.cdf'), - - ScienceFilePath('imap_glows_l3d_solar-hist_19470303-cr02306_v002.0001.cdf'), - AncillaryFilePath('imap_glows_uv-anis_19470303_20260110_v001.dat'), - AncillaryFilePath('imap_glows_lya_19470303_20260110_v001.dat'), - AncillaryFilePath('imap_glows_e-dens_19470303_20260110_v001.dat'), - AncillaryFilePath('imap_glows_p-dens_19470303_20260110_v001.dat'), - AncillaryFilePath('imap_glows_speed_19470303_20260110_v001.dat'), - AncillaryFilePath('imap_glows_phion_19470303_20260110_v001.dat'), - - ScienceFilePath('imap_glows_l3e_survival-probability-ul-sf_20251102-repoint00036_v006.0001.cdf'), - ScienceFilePath('imap_glows_l3e_survival-probability-ul-sf_20251115-repoint00049_v006.0001.cdf'), - AncillaryFilePath('imap_glows_survival-probability-ul-sf-raw_20251102-repoint00036_v001.dat'), - AncillaryFilePath('imap_glows_survival-probability-ul-sf-raw_20251115-repoint00049_v001.dat'), - - ScienceFilePath('imap_glows_l3e_survival-probability-ul-hf_20251102-repoint00036_v007.0001.cdf'), - ScienceFilePath('imap_glows_l3e_survival-probability-ul-hf_20251115-repoint00049_v007.0001.cdf'), - AncillaryFilePath('imap_glows_survival-probability-ul-hf-raw_20251102-repoint00036_v001.dat'), - AncillaryFilePath('imap_glows_survival-probability-ul-hf-raw_20251115-repoint00049_v001.dat'), - - ScienceFilePath('imap_glows_l3e_survival-probability-hi-45_20251102-repoint00036_v003.0001.cdf'), - ScienceFilePath('imap_glows_l3e_survival-probability-hi-45_20251115-repoint00049_v003.0001.cdf'), - AncillaryFilePath('imap_glows_survival-probability-hi-45-raw_20251102-repoint00036_v001.dat'), - AncillaryFilePath('imap_glows_survival-probability-hi-45-raw_20251115-repoint00049_v001.dat'), - - ScienceFilePath('imap_glows_l3e_survival-probability-hi-90_20251102-repoint00036_v004.0001.cdf'), - ScienceFilePath('imap_glows_l3e_survival-probability-hi-90_20251115-repoint00049_v004.0001.cdf'), - AncillaryFilePath('imap_glows_survival-probability-hi-90-raw_20251102-repoint00036_v001.dat'), - AncillaryFilePath('imap_glows_survival-probability-hi-90-raw_20251115-repoint00049_v001.dat'), - - ScienceFilePath('imap_glows_l3e_survival-probability-lo_20251102-repoint00036_v005.0001.cdf'), - ScienceFilePath('imap_glows_l3e_survival-probability-lo_20251115-repoint00049_v005.0001.cdf'), - AncillaryFilePath('imap_glows_survival-probability-lo-raw_20251102-repoint00036_v001.dat'), - AncillaryFilePath('imap_glows_survival-probability-lo-raw_20251115-repoint00049_v001.dat'), + ScienceFilePath( + "imap_glows_l3b_ion-rate-profile_20251102-cr02304_v000.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3b_ion-rate-profile_20251130-cr02305_v000.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3c_sw-profile_20251102-cr02304_v000.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3c_sw-profile_20251130-cr02305_v000.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3d_solar-hist_19470303-cr02306_v002.0001.cdf" + ), + AncillaryFilePath("imap_glows_uv-anis_19470303_20260110_v001.dat"), + AncillaryFilePath("imap_glows_lya_19470303_20260110_v001.dat"), + AncillaryFilePath("imap_glows_e-dens_19470303_20260110_v001.dat"), + AncillaryFilePath("imap_glows_p-dens_19470303_20260110_v001.dat"), + AncillaryFilePath("imap_glows_speed_19470303_20260110_v001.dat"), + AncillaryFilePath("imap_glows_phion_19470303_20260110_v001.dat"), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-sf_20251102-repoint00036_v006.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-sf_20251115-repoint00049_v006.0001.cdf" + ), + AncillaryFilePath( + "imap_glows_survival-probability-ul-sf-raw_20251102-repoint00036_v001.dat" + ), + AncillaryFilePath( + "imap_glows_survival-probability-ul-sf-raw_20251115-repoint00049_v001.dat" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-hf_20251102-repoint00036_v007.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-hf_20251115-repoint00049_v007.0001.cdf" + ), + AncillaryFilePath( + "imap_glows_survival-probability-ul-hf-raw_20251102-repoint00036_v001.dat" + ), + AncillaryFilePath( + "imap_glows_survival-probability-ul-hf-raw_20251115-repoint00049_v001.dat" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-45_20251102-repoint00036_v003.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-45_20251115-repoint00049_v003.0001.cdf" + ), + AncillaryFilePath( + "imap_glows_survival-probability-hi-45-raw_20251102-repoint00036_v001.dat" + ), + AncillaryFilePath( + "imap_glows_survival-probability-hi-45-raw_20251115-repoint00049_v001.dat" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-90_20251102-repoint00036_v004.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-90_20251115-repoint00049_v004.0001.cdf" + ), + AncillaryFilePath( + "imap_glows_survival-probability-hi-90-raw_20251102-repoint00036_v001.dat" + ), + AncillaryFilePath( + "imap_glows_survival-probability-hi-90-raw_20251115-repoint00049_v001.dat" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-lo_20251102-repoint00036_v005.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-lo_20251115-repoint00049_v005.0001.cdf" + ), + AncillaryFilePath( + "imap_glows_survival-probability-lo-raw_20251102-repoint00036_v001.dat" + ), + AncillaryFilePath( + "imap_glows_survival-probability-lo-raw_20251115-repoint00049_v001.dat" + ), ] for file_path in expected_files: - self.assertTrue(file_path.construct_path().exists(), msg=str(file_path.construct_path())) + self.assertTrue( + file_path.construct_path().exists(), + msg=str(file_path.construct_path()), + ) @run_periodically(timedelta(days=14)) @run_test_in_docker def test_l3bcde_automatic_reprocessing(self): - new_l3a_file = GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251118-repoint00052_v001.cdf" + new_l3a_file = ( + GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251118-repoint00052_v001.cdf" + ) l3bcde_input_files = [ GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251113-repoint00047_v001.cdf", GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251129-repoint00063_v001.cdf", @@ -373,41 +537,71 @@ def test_l3bcde_automatic_reprocessing(self): INTEGRATION_TEST_DATA / "spice" / "imap_recon_20250415_20260415_v01.bsp", ] - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) - with mock_imap_data_access(get_run_local_data_path("glows_reprocessing"), l3bcde_input_files): - processing_input = ProcessingInputCollection(RepointInput("imap_2026_090_01.repoint"), - AncillaryInput("imap_glows_force-reprocessing-config_20250101_v000.csv")) - input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version=VersionMap({}, Version(1, 0)), start_date=datetime(2000, 1, 1), - end_date=datetime(2000, 1, 1)) + with mock_imap_data_access( + get_run_local_data_path("glows_reprocessing"), l3bcde_input_files + ): + processing_input = ProcessingInputCollection( + RepointInput("imap_2026_090_01.repoint"), + AncillaryInput( + "imap_glows_force-reprocessing-config_20250101_v000.csv" + ), + ) + input_metadata = InputMetadata( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version=VersionMap({}, Version(1, 0)), + start_date=datetime(2000, 1, 1), + end_date=datetime(2000, 1, 1), + ) processor = GlowsProcessor(processing_input, input_metadata) processor.process() expected_files = [ - ScienceFilePath('imap_glows_l3b_ion-rate-profile_20251102-cr02304_v001.0002.cdf'), - - ScienceFilePath('imap_glows_l3c_sw-profile_20251102-cr02304_v001.0002.cdf'), - - ScienceFilePath('imap_glows_l3d_solar-hist_19470303-cr02306_v001.0002.cdf'), - AncillaryFilePath('imap_glows_uv-anis_19470303_20260110_v002.dat'), - AncillaryFilePath('imap_glows_lya_19470303_20260110_v002.dat'), - AncillaryFilePath('imap_glows_e-dens_19470303_20260110_v002.dat'), - AncillaryFilePath('imap_glows_p-dens_19470303_20260110_v002.dat'), - AncillaryFilePath('imap_glows_speed_19470303_20260110_v002.dat'), - AncillaryFilePath('imap_glows_phion_19470303_20260110_v002.dat'), - - ScienceFilePath("imap_glows_l3e_survival-probability-hi-45_20251102-repoint00036_v001.0001.cdf"), - ScienceFilePath("imap_glows_l3e_survival-probability-hi-90_20251102-repoint00036_v001.0001.cdf"), - ScienceFilePath("imap_glows_l3e_survival-probability-lo_20251102-repoint00036_v001.0001.cdf"), - ScienceFilePath("imap_glows_l3e_survival-probability-ul-hf_20251102-repoint00036_v001.0001.cdf"), - ScienceFilePath("imap_glows_l3e_survival-probability-ul-sf_20251102-repoint00036_v001.0001.cdf"), + ScienceFilePath( + "imap_glows_l3b_ion-rate-profile_20251102-cr02304_v001.0002.cdf" + ), + ScienceFilePath( + "imap_glows_l3c_sw-profile_20251102-cr02304_v001.0002.cdf" + ), + ScienceFilePath( + "imap_glows_l3d_solar-hist_19470303-cr02306_v001.0002.cdf" + ), + AncillaryFilePath("imap_glows_uv-anis_19470303_20260110_v002.dat"), + AncillaryFilePath("imap_glows_lya_19470303_20260110_v002.dat"), + AncillaryFilePath("imap_glows_e-dens_19470303_20260110_v002.dat"), + AncillaryFilePath("imap_glows_p-dens_19470303_20260110_v002.dat"), + AncillaryFilePath("imap_glows_speed_19470303_20260110_v002.dat"), + AncillaryFilePath("imap_glows_phion_19470303_20260110_v002.dat"), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-45_20251102-repoint00036_v001.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-hi-90_20251102-repoint00036_v001.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-lo_20251102-repoint00036_v001.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-hf_20251102-repoint00036_v001.0001.cdf" + ), + ScienceFilePath( + "imap_glows_l3e_survival-probability-ul-sf_20251102-repoint00036_v001.0001.cdf" + ), ] for file_path in expected_files: - self.assertTrue(file_path.construct_path().exists(), msg=str(file_path.construct_path())) + self.assertTrue( + file_path.construct_path().exists(), + msg=str(file_path.construct_path()), + ) @run_periodically(timedelta(days=14)) @run_test_in_docker @@ -426,7 +620,7 @@ def test_glows_l3abcde_from_l2(self): input_files = l2_paths + ancillary_file_paths l3a_integration_data_dir = get_run_local_data_path("glows_l3a_with_prod_l2") - with (mock_imap_data_access(l3a_integration_data_dir, input_files)): + with mock_imap_data_access(l3a_integration_data_dir, input_files): for i, l2_path in enumerate(l2_paths): l2_science_file_path = ScienceFilePath(l2_path) l2_science_input = ScienceInput(l2_path.name) @@ -443,9 +637,13 @@ def test_glows_l3abcde_from_l2(self): repointing=l2_science_file_path.repointing, ) - ancillary_inputs = [AncillaryInput(ancillary.name) for ancillary in ancillary_file_paths] + ancillary_inputs = [ + AncillaryInput(ancillary.name) for ancillary in ancillary_file_paths + ] processing_inputs = [l2_science_input, *ancillary_inputs] - processor = GlowsProcessor(ProcessingInputCollection(*processing_inputs), input_metadata) + processor = GlowsProcessor( + ProcessingInputCollection(*processing_inputs), input_metadata + ) try: _ = processor.process() @@ -479,23 +677,43 @@ def test_glows_l3abcde_from_l2(self): INTEGRATION_TEST_DATA / "spice" / "imap_recon_20250925_20260520_v01.bsp", ] - lo_l1b_folder = Path("/Users/harrison/Development/imap_L3_processing/data/imap/lo/l1b") + lo_l1b_folder = Path( + "/Users/harrison/Development/imap_L3_processing/data/imap/lo/l1b" + ) lo_l1b_inputs = list(lo_l1b_folder.rglob("*.cdf")) l3a_inputs = list((l3a_integration_data_dir / "imap/glows/l3a").rglob("*.cdf")) - repoint_file_path = [INTEGRATION_TEST_DATA / "spice" / "imap_2026_139_01.repoint"] + repoint_file_path = [ + INTEGRATION_TEST_DATA / "spice" / "imap_2026_139_01.repoint" + ] - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) - input_paths = l3bcde_ancillary_inputs + l3a_inputs + repoint_file_path + lo_l1b_inputs - with mock_imap_data_access(get_run_local_data_path("glows_l3bcde_with_prod_l2"), input_paths): - processing_input = ProcessingInputCollection(RepointInput("imap_2026_139_01.repoint"), - AncillaryInput("imap_glows_force-reprocessing-config_20250101_v000.csv")) - input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", - version=VersionMap({}, Version(None, 1)), start_date=datetime(2000, 1, 1), - end_date=datetime(2000, 1, 1)) + input_paths = ( + l3bcde_ancillary_inputs + l3a_inputs + repoint_file_path + lo_l1b_inputs + ) + with mock_imap_data_access( + get_run_local_data_path("glows_l3bcde_with_prod_l2"), input_paths + ): + processing_input = ProcessingInputCollection( + RepointInput("imap_2026_139_01.repoint"), + AncillaryInput( + "imap_glows_force-reprocessing-config_20250101_v000.csv" + ), + ) + input_metadata = InputMetadata( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version=VersionMap({}, Version(None, 1)), + start_date=datetime(2000, 1, 1), + end_date=datetime(2000, 1, 1), + ) processor = GlowsProcessor(processing_input, input_metadata) processor.process() @@ -503,16 +721,28 @@ def test_glows_l3abcde_from_l2(self): @skipIf(os.getenv("IMAP_API_KEY") is None, "Only runs with prod IMAP_API_KEY") @run_test_in_docker def test_run_glows_l3be_against_prod(self): - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) - imap_data_access.config["DATA_DIR"] = get_run_local_data_path("glows_l3bcde_prod") + imap_data_access.config["DATA_DIR"] = get_run_local_data_path( + "glows_l3bcde_prod" + ) - test_reprocessing_file = get_test_data_path("glows/imap_glows_force-reprocessing-config_20250101_v000.csv") - shutil.copy(test_reprocessing_file, AncillaryFilePath(test_reprocessing_file.name).construct_path()) + test_reprocessing_file = get_test_data_path( + "glows/imap_glows_force-reprocessing-config_20250101_v000.csv" + ) + shutil.copy( + test_reprocessing_file, + AncillaryFilePath(test_reprocessing_file.name).construct_path(), + ) repoint_file_name = "imap_2026_227_01.repoint" - processing_input = ProcessingInputCollection(RepointInput(repoint_file_name), AncillaryInput(test_reprocessing_file.name)) + processing_input = ProcessingInputCollection( + RepointInput(repoint_file_name), AncillaryInput(test_reprocessing_file.name) + ) descriptors_to_produce = [ "ion-rate-profile", @@ -531,11 +761,14 @@ def test_run_glows_l3be_against_prod(self): desc: { "major_version": 1, "minor_version": 1, - } for desc in descriptors_to_produce - } + } + for desc in descriptors_to_produce + }, } - with patch("imap_l3_data_processor._parse_cli_arguments") as mock_parse_cli_arguments: + with patch( + "imap_l3_data_processor._parse_cli_arguments" + ) as mock_parse_cli_arguments: mock_arguments = Mock() mock_arguments.instrument = "glows" mock_arguments.data_level = "l3b" @@ -551,8 +784,12 @@ def test_run_glows_l3be_against_prod(self): @staticmethod def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path: - official_l2_path = get_test_data_path("glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf") - json_file_path = get_test_instrument_team_data_path("glows/imap_glows_l2_20130908085214_orbX_modX_p_v00.json") + official_l2_path = get_test_data_path( + "glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf" + ) + json_file_path = get_test_instrument_team_data_path( + "glows/imap_glows_l2_20130908085214_orbX_modX_p_v00.json" + ) new_file_path = output_folder / official_l2_path.name new_file_path.unlink(missing_ok=True) @@ -561,14 +798,18 @@ def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path: with open(json_file_path) as f: instrument_data = json.load(f) - start_of_epoch_window = datetime.fromisoformat(instrument_data["start_time"]) - end_of_epoch_window = datetime.fromisoformat(instrument_data["end_time"]) + start_of_epoch_window = datetime.fromisoformat( + instrument_data["start_time"] + ) + end_of_epoch_window = datetime.fromisoformat( + instrument_data["end_time"] + ) epoch_window = end_of_epoch_window - start_of_epoch_window epoch = start_of_epoch_window + epoch_window / 2 cdf["epoch"][0] = epoch - cdf['start_time'][0] = instrument_data["start_time"] - cdf['end_time'][0] = instrument_data["end_time"] + cdf["start_time"][0] = instrument_data["start_time"] + cdf["end_time"][0] = instrument_data["end_time"] lightcurve_vars = [ "spin_angle", @@ -579,11 +820,21 @@ def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path: "ecliptic_lat", ] for var in lightcurve_vars: - cdf[var] = np.array(instrument_data["daily_lightcurve"][var])[np.newaxis, :] + cdf[var] = np.array(instrument_data["daily_lightcurve"][var])[ + np.newaxis, : + ] - cdf["raw_histograms"] = np.array(instrument_data["daily_lightcurve"]["raw_histogram"])[np.newaxis, :] + cdf["raw_histograms"] = np.array( + instrument_data["daily_lightcurve"]["raw_histogram"] + )[np.newaxis, :] cdf["histogram_flag_array"] = np.array( - [int(f, 16) for f in instrument_data["daily_lightcurve"]["histogram_flag_array"]])[np.newaxis, :] + [ + int(f, 16) + for f in instrument_data["daily_lightcurve"][ + "histogram_flag_array" + ] + ] + )[np.newaxis, :] single_value_vars = [ "filter_temperature_average", @@ -598,7 +849,7 @@ def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path: "spin_period_ground_std_dev", "position_angle_offset_average", "position_angle_offset_std_dev", - "identifier" + "identifier", ] for var in single_value_vars: cdf[var][0] = instrument_data[var] @@ -609,17 +860,23 @@ def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path: "spacecraft_velocity_average", "spacecraft_velocity_std_dev", "spin_axis_orientation_average", - "spin_axis_orientation_std_dev" + "spin_axis_orientation_std_dev", ] for var in vector_vars: cdf[var][0] = np.array(list(instrument_data[var].values())) - cdf["bad_time_flag_occurrences"][0] = list(instrument_data["bad_time_flag_occurences"].values()) - cdf["number_of_good_l1b_inputs"][0] = instrument_data["header"]["number_of_l1b_files_used"] - cdf["total_l1b_inputs"][0] = instrument_data["header"]["number_of_all_l1b_files"] + cdf["bad_time_flag_occurrences"][0] = list( + instrument_data["bad_time_flag_occurences"].values() + ) + cdf["number_of_good_l1b_inputs"][0] = instrument_data["header"][ + "number_of_l1b_files_used" + ] + cdf["total_l1b_inputs"][0] = instrument_data["header"][ + "number_of_all_l1b_files" + ] return new_file_path -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From f13387bd7f39379cc485f3b65a0c0390b8323d91 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 7 Oct 2026 17:17:55 -0400 Subject: [PATCH 3/3] tests/integration/test_glows_processor_integration.py: fail on docker build errors and create missing mount/data directories --- .../test_glows_processor_integration.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index e1f932d9..b31fa826 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -70,24 +70,34 @@ def decorated(self): else: l3_processing_dir = Path(tests.__file__).parent.parent - docker_build = subprocess.run( - [ - "docker", - "build", - "--platform", - "linux/amd64", - "-q", - "-f", - "Dockerfile_glows_integration", - ".", - ], - cwd=l3_processing_dir, - capture_output=True, - ) + try: + docker_build = subprocess.run( + [ + "docker", + "build", + "--platform", + "linux/amd64", + "-q", + "-f", + "Dockerfile_glows_integration", + ".", + ], + cwd=l3_processing_dir, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Failed to build docker container:\n{e.stderr.decode('utf-8')}" + ) from e image_hash = docker_build.stdout.strip().decode("utf-8") print(f"Built docker container: {image_hash}") + # Bind mounts fail if the source directory does not exist, e.g. in a fresh checkout + for mounted_dir in ["temp_cdf_data", "run_local_input_data"]: + (l3_processing_dir / mounted_dir).mkdir(exist_ok=True) + args = [ "docker", "run", @@ -734,10 +744,11 @@ def test_run_glows_l3be_against_prod(self): test_reprocessing_file = get_test_data_path( "glows/imap_glows_force-reprocessing-config_20250101_v000.csv" ) - shutil.copy( - test_reprocessing_file, - AncillaryFilePath(test_reprocessing_file.name).construct_path(), - ) + reprocessing_file_destination = AncillaryFilePath( + test_reprocessing_file.name + ).construct_path() + reprocessing_file_destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(test_reprocessing_file, reprocessing_file_destination) repoint_file_name = "imap_2026_227_01.repoint" processing_input = ProcessingInputCollection(