diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index b31fa826..ae8f2b15 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -6,7 +6,7 @@ import subprocess import unittest from datetime import timedelta, datetime -from functools import wraps +from functools import wraps, partial from pathlib import Path from typing import Callable from unittest import skipIf @@ -29,6 +29,10 @@ create_glows_l3a_from_dictionary, ) from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT +from imap_l3_processing.glows.l3e.glows_l3e_initializer import ( + identify_versions_for_l3e_output_files, +) +from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.integration.integration_test_helpers import ( @@ -110,8 +114,9 @@ def decorated(self): f"type=bind,src={l3_processing_dir}/run_local_input_data,dst=/run_local_input_data", ] - if imap_api_key := os.getenv("IMAP_API_KEY"): - args += ["-e", f"IMAP_API_KEY={imap_api_key}"] + # Pass by name so docker reads the value from the environment and the key isn't in error messages + if os.getenv("IMAP_API_KEY"): + args += ["-e", "IMAP_API_KEY"] if imap_data_access_url := os.getenv("IMAP_DATA_ACCESS_URL"): args += ["-e", f"IMAP_DATA_ACCESS_URL={imap_data_access_url}"] @@ -123,6 +128,29 @@ def decorated(self): return decorated +def only_process_l3e_for(repointings_to_keep: set[int]): + # Each repointing runs five survival probability executables and needs SPICE kernels covering it, + # so only select the repointings the test checks + def identify_subset(*args, **kwargs) -> GlowsL3eVersionsForRepointings: + def keep(versions: dict) -> dict: + return {n: v for n, v in versions.items() if n in repointings_to_keep} + + r = identify_versions_for_l3e_output_files(*args, **kwargs) + return GlowsL3eVersionsForRepointings( + [n for n in r.repointing_numbers if n in repointings_to_keep], + keep(r.hi_90_repointings), + keep(r.hi_45_repointings), + keep(r.lo_repointings), + keep(r.ultra_sf_repointings), + keep(r.ultra_hf_repointings), + ) + + return patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files", + side_effect=identify_subset, + ) + + class TestGlowsProcessorIntegration(unittest.TestCase): @with_tempdir def test_glows_l3a(self, tmp_dir): @@ -399,7 +427,8 @@ def test_l3bcde_first_time_processing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - processor.process() + with only_process_l3e_for({36, 49}): + processor.process() expected_files = [ ScienceFilePath( @@ -572,7 +601,8 @@ def test_l3bcde_automatic_reprocessing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - processor.process() + with only_process_l3e_for({36}): + processor.process() expected_files = [ ScienceFilePath( @@ -770,16 +800,24 @@ def test_run_glows_l3be_against_prod(self): "dependency": json.loads(processing_input.serialize()), "version": { desc: { - "major_version": 1, + "major_version": 2, "minor_version": 1, } for desc in descriptors_to_produce }, } - with patch( - "imap_l3_data_processor._parse_cli_arguments" - ) as mock_parse_cli_arguments: + # Nothing is uploaded, so reruns compute the same output names as earlier local runs; overwrite them + with ( + patch( + "imap_l3_data_processor._parse_cli_arguments" + ) as mock_parse_cli_arguments, + patch( + "imap_l3_processing.glows.glows_processor.save_data", + side_effect=partial(save_data, delete_if_present=True), + ), + only_process_l3e_for({36}), + ): mock_arguments = Mock() mock_arguments.instrument = "glows" mock_arguments.data_level = "l3b" diff --git a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv index 29c5c783..c307eb12 100644 --- a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv +++ b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv @@ -1,6 +1,6 @@ -ignore-after: 2026-08-13 12:39:07.208204+00:00 -l3e_survival-probability-hi-45 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-hi-90 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-lo repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-ul-hf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-ul-sf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 \ No newline at end of file +ignore-after: 2100-01-01 00:00:00+00:00 +l3e_survival-probability-hi-45 repoint36 +l3e_survival-probability-hi-90 repoint36 +l3e_survival-probability-lo repoint36 +l3e_survival-probability-ul-hf repoint36 +l3e_survival-probability-ul-sf repoint36 \ No newline at end of file