From 529535b81ac659f71e9cb0a62824916fa78c7b6d Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Tue, 6 Oct 2026 14:36:47 -0400 Subject: [PATCH 1/5] support corrected spin start times (use_corrected_spin_start, default True) --- imap_processing/spice/spin.py | 117 ++++++++++++--- imap_processing/tests/conftest.py | 8 + .../tests/spice/test_data/fake_spin_data.csv | 24 +-- imap_processing/tests/spice/test_spin.py | 139 ++++++++++++++++-- 4 files changed, 248 insertions(+), 40 deletions(-) diff --git a/imap_processing/spice/spin.py b/imap_processing/spice/spin.py index 20264e4739..883787c62e 100644 --- a/imap_processing/spice/spin.py +++ b/imap_processing/spice/spin.py @@ -37,13 +37,21 @@ def set_global_spin_table_paths(paths: list[Path]) -> None: config._spin_table_paths = paths -def get_spin_data() -> pd.DataFrame: +def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: """ Read spin-tables and return spin data. The spin-tables to read are stored in the mutable module level attribute named `spin_table_paths`. + Parameters + ---------- + use_corrected_spin_start : bool + If True (default), use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). + Returns ------- spin_data : pandas.DataFrame @@ -52,11 +60,19 @@ def get_spin_data() -> pd.DataFrame: * `spin_number`: Unique integer spin number. * `spin_start_sec_sclk`: MET seconds of spin start time. * `spin_start_subsec_sclk`: MET microseconds of spin start time. - * `spin_start_met`: Floating point MET seconds of spin start. + * `spin_start_met`: Floating point MET seconds of spin start, from + the corrected or original spin start time per + `use_corrected_spin_start`. * `spin_start_utc`: UTC string of spin start time. + * `spin_start_sec_sclk_corr`: MET seconds of the corrected spin start + time. + * `spin_start_subsec_sclk_corr`: MET microseconds of the corrected + spin start time. + * `spin_start_utc_corr`: UTC string of the corrected spin start time. * `spin_period_sec`: Floating point spin period in seconds (estimated). * `actual_spin_period`: Floating point actual spin period computed from - consecutive spin start times. More accurate than spin_period_sec. + consecutive `spin_start_met` values. More accurate than + spin_period_sec. * `spin_period_valid`: Boolean indicating whether spin period is valid. * `spin_phase_valid`: Boolean indicating whether spin phase is valid. * `spin_period_source`: Source used for determining spin period. @@ -74,11 +90,15 @@ def get_spin_data() -> pd.DataFrame: "module attribute spin_table_paths." ) - return _load_spin_data_with_cache(tuple(config._spin_table_paths)) + return _load_spin_data_with_cache( + tuple(config._spin_table_paths), use_corrected_spin_start + ) @functools.cache -def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: +def _load_spin_data_with_cache( + csv_paths: tuple[Path], use_corrected_spin_start: bool +) -> pd.DataFrame: """ Load spin-table data from csv files and combine them. @@ -86,16 +106,29 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: ---------- csv_paths : tuple[Path] Locations of spin-table csv files. + use_corrected_spin_start : bool + If True, use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). Returns ------- combined_df: pandas.DataFrame The dataframe containing all spin data. + + Raises + ------ + ValueError + If a spin-table lacks the corrected spin start time columns. """ logger.debug( f"Merging the following spin tables files: {[sp.name for sp in csv_paths]}" ) + # Reversed sorting is used so that the proper precedence is applied in + # the below use of DataFrame.combine_first() + sorted_paths = sorted(csv_paths, reverse=True) spin_dataframes = [ pd.read_csv( spin_table_path, @@ -106,6 +139,9 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: "spin_start_sec_sclk": int, "spin_start_subsec_sclk": int, "spin_start_utc": str, + "spin_start_sec_sclk_corr": int, + "spin_start_subsec_sclk_corr": int, + "spin_start_utc_corr": str, "spin_period_sec": float, "spin_period_valid": bool, "spin_phase_valid": bool, @@ -113,20 +149,35 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: "thruster_firing": bool, }, ) - # Reversed sorting is used so that the proper precedence is applied in - # the below use of DataFrame.combine_first() - for spin_table_path in sorted(csv_paths, reverse=True) + for spin_table_path in sorted_paths + ] + corrected_columns = [ + "spin_start_sec_sclk_corr", + "spin_start_subsec_sclk_corr", + "spin_start_utc_corr", ] + for spin_table_path, spin_dataframe in zip( + sorted_paths, spin_dataframes, strict=True + ): + missing = [col for col in corrected_columns if col not in spin_dataframe] + if missing: + raise ValueError( + f"Spin table {spin_table_path.name} is missing the corrected spin " + f"start time columns {missing}." + ) combined_df = reduce( lambda left, right: left.combine_first(right), spin_dataframes, ) # Duplicate the index so that users can access "spin_numer" by name combined_df.insert(0, "spin_number", combined_df.index) - # Combine spin_start_sec_sclk and spin_start_subsec_sclk to get the spin start - # time in seconds. The spin start subseconds are in microseconds. + # Combine spin_start_sec_sclk and spin_start_subsec_sclk (or their corrected + # versions) to get the spin start time in seconds. The spin start subseconds + # are in microseconds. + suffix = "_corr" if use_corrected_spin_start else "" combined_df["spin_start_met"] = ( - combined_df["spin_start_sec_sclk"] + combined_df["spin_start_subsec_sclk"] / 1e6 + combined_df[f"spin_start_sec_sclk{suffix}"] + + combined_df[f"spin_start_subsec_sclk{suffix}"] / 1e6 ) # Precompute actual spin periods from consecutive spin start times # Only use actual periods when spin numbers increment by exactly 1 @@ -147,7 +198,9 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: return combined_df -def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: +def interpolate_spin_data( + query_met_times: float | npt.NDArray, use_corrected_spin_start: bool = True +) -> pd.DataFrame: """ Interpolate spin table data to the queried MET times. @@ -160,6 +213,11 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: ---------- query_met_times : float or np.ndarray Query times in Mission Elapsed Time (MET). + use_corrected_spin_start : bool + If True (default), use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). Returns ------- @@ -168,7 +226,7 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: the columns output from :py:func:`get_spin_data`, the `sc_spin_phase` column is added and is uniquely computed for each queried MET time. """ - spin_df = get_spin_data() + spin_df = get_spin_data(use_corrected_spin_start) # Ensure query_met_times is an array query_met_times = np.asarray(query_met_times) @@ -237,7 +295,9 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: return out_df -def get_spin_number(met_time: float | npt.NDArray) -> int | npt.NDArray: +def get_spin_number( + met_time: float | npt.NDArray, use_corrected_spin_start: bool = True +) -> int | npt.NDArray: """ Get the spin number for the input query time. @@ -248,13 +308,18 @@ def get_spin_number(met_time: float | npt.NDArray) -> int | npt.NDArray: ---------- met_time : float or np.ndarray Query time in Mission Elapsed Time (MET). + use_corrected_spin_start : bool + If True (default), use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). Returns ------- spin_number : int or np.ndarray Spin number for the input query time. """ - spin_df = interpolate_spin_data(met_time) + spin_df = interpolate_spin_data(met_time, use_corrected_spin_start) spin_numbers = spin_df["spin_number"].values return spin_numbers.item() if np.asarray(met_time).ndim == 0 else spin_numbers @@ -296,7 +361,7 @@ def get_spin_angle( def get_spacecraft_spin_phase( - query_met_times: float | npt.NDArray, + query_met_times: float | npt.NDArray, use_corrected_spin_start: bool = True ) -> float | npt.NDArray: """ Get the spacecraft spin phase for the input query times. @@ -308,20 +373,27 @@ def get_spacecraft_spin_phase( ---------- query_met_times : float or np.ndarray Query times in Mission Elapsed Time (MET). + use_corrected_spin_start : bool + If True (default), use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). Returns ------- spin_phase : float or np.ndarray Spin phase for the input query times. """ - spin_df = interpolate_spin_data(query_met_times) + spin_df = interpolate_spin_data(query_met_times, use_corrected_spin_start) if np.asarray(query_met_times).ndim == 0: return spin_df["sc_spin_phase"].values[0] return spin_df["sc_spin_phase"].values def get_instrument_spin_phase( - query_met_times: float | npt.NDArray, instrument: SpiceFrame + query_met_times: float | npt.NDArray, + instrument: SpiceFrame, + use_corrected_spin_start: bool = True, ) -> float | npt.NDArray: """ Get the instrument spin phase for the input query times. @@ -335,6 +407,11 @@ def get_instrument_spin_phase( Query times in Mission Elapsed Time (MET). instrument : SpiceFrame Instrument frame to calculate spin phase for. + use_corrected_spin_start : bool + If True (default), use the corrected spin start times + (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise + use the original spin start times (`spin_start_sec_sclk`, + `spin_start_subsec_sclk`). Returns ------- @@ -343,7 +420,9 @@ def get_instrument_spin_phase( floating point number in the range [0, 1) corresponding to the spin angle / 360. """ - spacecraft_spin_phase = get_spacecraft_spin_phase(query_met_times) + spacecraft_spin_phase = get_spacecraft_spin_phase( + query_met_times, use_corrected_spin_start + ) instrument_spin_phase_offset = get_spacecraft_to_instrument_spin_phase_offset( instrument ) diff --git a/imap_processing/tests/conftest.py b/imap_processing/tests/conftest.py index 9bae21b7b3..748649696e 100644 --- a/imap_processing/tests/conftest.py +++ b/imap_processing/tests/conftest.py @@ -329,6 +329,14 @@ def make_data( nspins, spin_start_subsec, dtype=np.uint32 ), "spin_start_utc": np.datetime_as_string(spin_start_dt64, unit="us"), + # No correction to the spin start times + "spin_start_sec_sclk_corr": spin_start_sec, + "spin_start_subsec_sclk_corr": np.full( + nspins, spin_start_subsec, dtype=np.uint32 + ), + "spin_start_utc_corr": np.datetime_as_string( + spin_start_dt64, unit="us" + ), "spin_period_sec": np.full(nspins, spin_period, dtype=np.float32), "spin_period_valid": np.ones(nspins, dtype=np.uint8), "spin_phase_valid": np.ones(nspins, dtype=np.uint8), diff --git a/imap_processing/tests/spice/test_data/fake_spin_data.csv b/imap_processing/tests/spice/test_data/fake_spin_data.csv index 9a25eb9199..d83aea7252 100644 --- a/imap_processing/tests/spice/test_data/fake_spin_data.csv +++ b/imap_processing/tests/spice/test_data/fake_spin_data.csv @@ -1,21 +1,21 @@ # This is a fake csv file for the sole purpose of testing the get_spacecraft_spin_phase function -spin_number,spin_start_sec_sclk,spin_start_subsec_sclk,spin_start_utc,spin_period_sec,spin_period_valid,spin_phase_valid,spin_period_source,thruster_firing +spin_number,spin_start_sec_sclk,spin_start_subsec_sclk,spin_start_utc,spin_start_sec_sclk_corr,spin_start_subsec_sclk_corr,spin_start_utc_corr,spin_period_sec,spin_period_valid,spin_phase_valid,spin_period_source,thruster_firing # Start with thruster firing -0,0,0,2024-04-11 00:00:00.000000,15.0,1,1,0,1 +0,0,0,2024-04-11 00:00:00.000000,0,0,2024-04-11 00:00:00.000000,15.0,1,1,0,1 # Turn thruster firing off then 3 valid spins -1,15,0,2024-04-11 00:00:15.000000,15.0,1,1,0,0 -2,30,0,2024-04-11 00:00:30.000000,15.0,1,1,0,0 -3,45,0,2024-04-11 00:00:45.000000,15.0,1,1,0,0 +1,15,0,2024-04-11 00:00:15.000000,15,0,2024-04-11 00:00:15.000000,15.0,1,1,0,0 +2,30,0,2024-04-11 00:00:30.000000,30,0,2024-04-11 00:00:30.000000,15.0,1,1,0,0 +3,45,0,2024-04-11 00:00:45.000000,45,0,2024-04-11 00:00:45.000000,15.0,1,1,0,0 # Missing spin then 2 good spins -5,75,0,2024-04-11 00:01:15.000000,15.0,1,1,0,0 -6,90,0,2024-04-11 00:01:30.000000,15.0,1,1,0,0 +5,75,0,2024-04-11 00:01:15.000000,75,0,2024-04-11 00:01:15.000000,15.0,1,1,0,0 +6,90,0,2024-04-11 00:01:30.000000,90,0,2024-04-11 00:01:30.000000,15.0,1,1,0,0 # invalid spin period -7,105,0,2024-04-11 00:01:45.000000,15.0,0,1,0,0 +7,105,0,2024-04-11 00:01:45.000000,105,0,2024-04-11 00:01:45.000000,15.0,0,1,0,0 # invalid spin phase -8,120,0,2024-04-11 00:02:00.000000,15.0,1,0,0,0 +8,120,0,2024-04-11 00:02:00.000000,120,0,2024-04-11 00:02:00.000000,15.0,1,0,0,0 # 1 good spin with slightly shorter period estimate -9,135,0,2024-04-11 00:02:15.000000,14.5,1,1,0,0 +9,135,0,2024-04-11 00:02:15.000000,135,0,2024-04-11 00:02:15.000000,14.5,1,1,0,0 # Spin that starts 0.5s after estimated end of spin 9 (tests gap handling) -10,150,0,2024-04-11 00:02:30.000000,15.0,1,1,0,0 +10,150,0,2024-04-11 00:02:30.000000,150,0,2024-04-11 00:02:30.000000,15.0,1,1,0,0 # Thruster firing on -11,165,0,2024-04-11 00:02:45.000000,15.0,1,1,0,1 \ No newline at end of file +11,165,0,2024-04-11 00:02:45.000000,165,0,2024-04-11 00:02:45.000000,15.0,1,1,0,1 \ No newline at end of file diff --git a/imap_processing/tests/spice/test_spin.py b/imap_processing/tests/spice/test_spin.py index 808e093dc9..80c84e7e05 100644 --- a/imap_processing/tests/spice/test_spin.py +++ b/imap_processing/tests/spice/test_spin.py @@ -6,7 +6,9 @@ import pytest from imap_processing.spice import config, spin -from imap_processing.spice.geometry import SpiceFrame +from imap_processing.spice.geometry import ( + SpiceFrame, +) @pytest.fixture @@ -42,6 +44,9 @@ def test_set_spin_table_paths(monkeypatch): 15, 0, "2024-04-11 00:00:15.000000", + 15, + 0, + "2024-04-11 00:00:15.000000", 15.0, True, True, @@ -61,6 +66,9 @@ def test_set_spin_table_paths(monkeypatch): 15, 0, "2024-04-11 00:00:15.000000", + 15, + 0, + "2024-04-11 00:00:15.000000", 15.0, True, True, @@ -75,6 +83,9 @@ def test_set_spin_table_paths(monkeypatch): 30, 0, "2024-04-11 00:00:30.000000", + 30, + 0, + "2024-04-11 00:00:30.000000", 15.0, True, True, @@ -225,6 +236,9 @@ def test_get_spin_data(use_fake_spin_data_for_time): "thruster_firing", "spin_start_met", "actual_spin_period", + "spin_start_sec_sclk_corr", + "spin_start_subsec_sclk_corr", + "spin_start_utc_corr", }, "Spin data must have the specified fields." @@ -235,6 +249,9 @@ def test_get_spin_table_merge(tmp_path, use_test_spin_data_csv): "spin_start_sec_sclk", "spin_start_subsec_sclk", "spin_start_utc", + "spin_start_sec_sclk_corr", + "spin_start_subsec_sclk_corr", + "spin_start_utc_corr", "spin_period_sec", "spin_period_valid", "spin_phase_valid", @@ -243,10 +260,62 @@ def test_get_spin_table_merge(tmp_path, use_test_spin_data_csv): ] # Table 1 is missing spin # 2 table1_data = [ - [0, 0, 0, "2025-05-01 00:00:00.000", 15, 1, 1, 0, 0], - [1, 15, 0, "2025-05-01 00:00:15.000", 15, 1, 1, 0, 0], - [3, 45, 0, "2025-05-01 00:00:45.000", 15, 1, 1, 0, 0], - [4, 60, 0, "2025-05-01 00:01:00.000", 15, 1, 1, 0, 0], + [ + 0, + 0, + 0, + "2025-05-01 00:00:00.000", + 0, + 0, + "2025-05-01 00:00:00.000", + 15, + 1, + 1, + 0, + 0, + ], + [ + 1, + 15, + 0, + "2025-05-01 00:00:15.000", + 15, + 0, + "2025-05-01 00:00:15.000", + 15, + 1, + 1, + 0, + 0, + ], + [ + 3, + 45, + 0, + "2025-05-01 00:00:45.000", + 45, + 0, + "2025-05-01 00:00:45.000", + 15, + 1, + 1, + 0, + 0, + ], + [ + 4, + 60, + 0, + "2025-05-01 00:01:00.000", + 60, + 0, + "2025-05-01 00:01:00.000", + 15, + 1, + 1, + 0, + 0, + ], ] table1_path = tmp_path / "imap_2025_100_2025_101_01.spin.csv" pd.DataFrame.from_records(table1_data, columns=columns, index=columns[0]).to_csv( @@ -254,10 +323,62 @@ def test_get_spin_table_merge(tmp_path, use_test_spin_data_csv): ) # Table 2 fills in spin #2 and changes values for spin #3 table2_data = [ - [2, 30, 0, "2025-05-01 00:00:30.000", 15.1, 1, 1, 0, 0], - [3, 45, 1e5, "2025-05-01 00:00:45.100", 14.9, 1, 1, 0, 0], - [5, 75, 0, "2025-05-01 00:01:15.000", 15, 1, 1, 0, 0], - [6, 90, 0, "2025-05-01 00:01:30.000", 15, 1, 1, 0, 0], + [ + 2, + 30, + 0, + "2025-05-01 00:00:30.000", + 30, + 0, + "2025-05-01 00:00:30.000", + 15.1, + 1, + 1, + 0, + 0, + ], + [ + 3, + 45, + 1e5, + "2025-05-01 00:00:45.100", + 45, + 1e5, + "2025-05-01 00:00:45.100", + 14.9, + 1, + 1, + 0, + 0, + ], + [ + 5, + 75, + 0, + "2025-05-01 00:01:15.000", + 75, + 0, + "2025-05-01 00:01:15.000", + 15, + 1, + 1, + 0, + 0, + ], + [ + 6, + 90, + 0, + "2025-05-01 00:01:30.000", + 90, + 0, + "2025-05-01 00:01:30.000", + 15, + 1, + 1, + 0, + 0, + ], ] table2_path = tmp_path / "imap_2025_101_2025_102_01.spin.csv" pd.DataFrame.from_records(table2_data, columns=columns, index=columns[0]).to_csv( From 05be1341d4fc45abeb8ee343c4bb97f5331b617c Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Tue, 6 Oct 2026 17:08:02 -0400 Subject: [PATCH 2/5] not using stale data when a packaged data file (e.g. in tests/spice/test_data) has changed --- .github/workflows/test.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d471dcd9e4..7016762d3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,9 @@ jobs: matrix: os: [windows-latest, ubuntu-latest, macos-latest] python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + env: + # External test data is only downloaded (and cached) on this one runner + USE_EXTERNAL_DATA: ${{ contains(matrix.os, 'ubuntu') && matrix.python-version == '3.10' }} defaults: run: shell: bash @@ -44,7 +47,7 @@ jobs: uses: actions/cache@v6 # We only download data on this runner, so only cache data from this run # so we don't get an empty cache on other runners overriding that data - if: ${{ contains(matrix.os, 'ubuntu') && matrix.python-version == '3.10' }} + if: env.USE_EXTERNAL_DATA == 'true' with: path: | **/data/ @@ -53,10 +56,15 @@ jobs: restore-keys: | cdf-data- + - name: Restore tracked files overwritten by the cache + # A fallback restore via restore-keys can bring back stale copies of + # git-tracked files under data/ and test_data/; reset them to the + # checked-out versions while keeping the untracked downloaded data. + if: env.USE_EXTERNAL_DATA == 'true' + run: git checkout -- . + - name: Testing id: test - env: - USE_EXTERNAL_DATA: ${{ contains(matrix.os, 'ubuntu') && matrix.python-version == '3.10' }} run: | # Ignore the network marks from the remote test environment if [ "$USE_EXTERNAL_DATA" = "true" ]; then From 31681b36c8043e85361aeba2490f2724aaf7c161 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Wed, 7 Oct 2026 16:06:44 -0400 Subject: [PATCH 3/5] get_spin_data and interpolate_spin_data return their original columns along with new ones --- imap_processing/spice/spin.py | 225 +++++++++++------------ imap_processing/tests/spice/test_spin.py | 20 +- 2 files changed, 126 insertions(+), 119 deletions(-) diff --git a/imap_processing/spice/spin.py b/imap_processing/spice/spin.py index 883787c62e..383263a828 100644 --- a/imap_processing/spice/spin.py +++ b/imap_processing/spice/spin.py @@ -37,21 +37,13 @@ def set_global_spin_table_paths(paths: list[Path]) -> None: config._spin_table_paths = paths -def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: +def get_spin_data() -> pd.DataFrame: """ Read spin-tables and return spin data. The spin-tables to read are stored in the mutable module level attribute named `spin_table_paths`. - Parameters - ---------- - use_corrected_spin_start : bool - If True (default), use the corrected spin start times - (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise - use the original spin start times (`spin_start_sec_sclk`, - `spin_start_subsec_sclk`). - Returns ------- spin_data : pandas.DataFrame @@ -60,9 +52,7 @@ def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: * `spin_number`: Unique integer spin number. * `spin_start_sec_sclk`: MET seconds of spin start time. * `spin_start_subsec_sclk`: MET microseconds of spin start time. - * `spin_start_met`: Floating point MET seconds of spin start, from - the corrected or original spin start time per - `use_corrected_spin_start`. + * `spin_start_met`: Floating point MET seconds of spin start. * `spin_start_utc`: UTC string of spin start time. * `spin_start_sec_sclk_corr`: MET seconds of the corrected spin start time. @@ -71,12 +61,15 @@ def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: * `spin_start_utc_corr`: UTC string of the corrected spin start time. * `spin_period_sec`: Floating point spin period in seconds (estimated). * `actual_spin_period`: Floating point actual spin period computed from - consecutive `spin_start_met` values. More accurate than - spin_period_sec. + consecutive spin start times. More accurate than spin_period_sec. * `spin_period_valid`: Boolean indicating whether spin period is valid. * `spin_phase_valid`: Boolean indicating whether spin phase is valid. * `spin_period_source`: Source used for determining spin period. * `thruster_firing`: Boolean indicating whether thruster is firing. + * `spin_start_met_corr`: Floating point MET seconds of the corrected + spin start. + * `actual_spin_period_corr`: Floating point actual spin period computed + from consecutive corrected spin start times. Raises ------ @@ -90,15 +83,11 @@ def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: "module attribute spin_table_paths." ) - return _load_spin_data_with_cache( - tuple(config._spin_table_paths), use_corrected_spin_start - ) + return _load_spin_data_with_cache(tuple(config._spin_table_paths)) @functools.cache -def _load_spin_data_with_cache( - csv_paths: tuple[Path], use_corrected_spin_start: bool -) -> pd.DataFrame: +def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: """ Load spin-table data from csv files and combine them. @@ -106,11 +95,6 @@ def _load_spin_data_with_cache( ---------- csv_paths : tuple[Path] Locations of spin-table csv files. - use_corrected_spin_start : bool - If True, use the corrected spin start times - (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise - use the original spin start times (`spin_start_sec_sclk`, - `spin_start_subsec_sclk`). Returns ------- @@ -171,36 +155,37 @@ def _load_spin_data_with_cache( ) # Duplicate the index so that users can access "spin_numer" by name combined_df.insert(0, "spin_number", combined_df.index) - # Combine spin_start_sec_sclk and spin_start_subsec_sclk (or their corrected - # versions) to get the spin start time in seconds. The spin start subseconds - # are in microseconds. - suffix = "_corr" if use_corrected_spin_start else "" - combined_df["spin_start_met"] = ( - combined_df[f"spin_start_sec_sclk{suffix}"] - + combined_df[f"spin_start_subsec_sclk{suffix}"] / 1e6 - ) - # Precompute actual spin periods from consecutive spin start times - # Only use actual periods when spin numbers increment by exactly 1 - # This prevents invalid times from appearing valid when spins are missing spin_numbers = combined_df["spin_number"].values - spin_number_diffs = np.diff(spin_numbers) - time_diffs = np.diff(combined_df["spin_start_met"].values) - - # Use actual time diff only where spin numbers increment by 1 - # Otherwise use the estimated spin_period_sec - actual_spin_periods = np.where( - spin_number_diffs == 1, time_diffs, combined_df["spin_period_sec"].values[:-1] - ) - # For the last spin, use the provided spin_period_sec since there's no next spin - combined_df["actual_spin_period"] = np.append( - actual_spin_periods, combined_df["spin_period_sec"].values[-1] - ) + spin_periods_sec = combined_df["spin_period_sec"].values + # Compute spin_start_met / actual_spin_period from the original spin start + # times, and spin_start_met_corr / actual_spin_period_corr from the corrected + # ones. + for suffix in ("", "_corr"): + # Combine spin_start_sec_sclk and spin_start_subsec_sclk to get the spin + # start time in seconds. The spin start subseconds are in microseconds. + spin_start_met = ( + combined_df[f"spin_start_sec_sclk{suffix}"] + + combined_df[f"spin_start_subsec_sclk{suffix}"] / 1e6 + ) + combined_df[f"spin_start_met{suffix}"] = spin_start_met + # Precompute actual spin periods from consecutive spin start times + # Only use actual periods when spin numbers increment by exactly 1 + # This prevents invalid times from appearing valid when spins are missing + # Otherwise use the estimated spin_period_sec + actual_spin_periods = np.where( + np.diff(spin_numbers) == 1, + np.diff(spin_start_met.values), + spin_periods_sec[:-1], + ) + # For the last spin, use the provided spin_period_sec since there's no + # next spin + combined_df[f"actual_spin_period{suffix}"] = np.append( + actual_spin_periods, spin_periods_sec[-1] + ) return combined_df -def interpolate_spin_data( - query_met_times: float | npt.NDArray, use_corrected_spin_start: bool = True -) -> pd.DataFrame: +def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: """ Interpolate spin table data to the queried MET times. @@ -209,24 +194,26 @@ def interpolate_spin_data( spin phase at the queried MET times. Note that spin phase is by definition, in the interval [0, 1) where 1 is equivalent to 360 degrees. + Rows are selected, and `sc_spin_phase` computed, using the original spin + start times. The `spin_number_corr` and `sc_spin_phase_corr` columns give + the spin number and spin phase computed using the corrected spin start times + instead; these are what :py:func:`get_spin_number` and + :py:func:`get_spacecraft_spin_phase` return. + Parameters ---------- query_met_times : float or np.ndarray Query times in Mission Elapsed Time (MET). - use_corrected_spin_start : bool - If True (default), use the corrected spin start times - (`spin_start_sec_sclk_corr`, `spin_start_subsec_sclk_corr`); otherwise - use the original spin start times (`spin_start_sec_sclk`, - `spin_start_subsec_sclk`). Returns ------- spin_df : pandas.DataFrame Spin table data interpolated for each queried MET time. In addition to - the columns output from :py:func:`get_spin_data`, the `sc_spin_phase` - column is added and is uniquely computed for each queried MET time. + the columns output from :py:func:`get_spin_data`, the `sc_spin_phase`, + `spin_number_corr` and `sc_spin_phase_corr` columns are added and are + uniquely computed for each queried MET time. """ - spin_df = get_spin_data(use_corrected_spin_start) + spin_df = get_spin_data() # Ensure query_met_times is an array query_met_times = np.asarray(query_met_times) @@ -236,61 +223,65 @@ def interpolate_spin_data( # convert scalar to array query_met_times = np.atleast_1d(query_met_times) - # Cache frequently accessed arrays to avoid repeated .values calls - spin_start_met = spin_df["spin_start_met"].values - actual_spin_periods = spin_df["actual_spin_period"].values + # Compute the spin number and spin phase using the original spin start + # times, then using the corrected ones. + for suffix in ("", "_corr"): + # Cache frequently accessed arrays to avoid repeated .values calls + spin_start_met = spin_df[f"spin_start_met{suffix}"].values + actual_spin_periods = spin_df[f"actual_spin_period{suffix}"].values + + # Make sure input times are within the bounds of spin data + spin_df_start_time = spin_start_met[0] + spin_df_end_time = spin_start_met[-1] + actual_spin_periods[-1] + input_start_time = query_met_times.min() + input_end_time = query_met_times.max() + if input_start_time < spin_df_start_time or input_end_time >= spin_df_end_time: + raise ValueError( + f"Query times, {query_met_times} are outside of the spin data range, " + f"{spin_df_start_time, spin_df_end_time}." + ) - # Make sure input times are within the bounds of spin data - spin_df_start_time = spin_start_met[0] - spin_df_end_time = spin_start_met[-1] + actual_spin_periods[-1] - input_start_time = query_met_times.min() - input_end_time = query_met_times.max() - if input_start_time < spin_df_start_time or input_end_time >= spin_df_end_time: - raise ValueError( - f"Query times, {query_met_times} are outside of the spin data range, " - f"{spin_df_start_time, spin_df_end_time}." + # Find all spin time that are less or equal to query_met_times. + # To do that, use side right, a[i-1] <= v < a[i], in the searchsorted. + # Eg. + # >>> df['a'] + # array([0, 15, 30, 45, 60]) + # >>> np.searchsorted(df['a'], [0, 13, 15, 32, 70], side='right') + # array([1, 1, 2, 3, 5]) + last_spin_indices = ( + np.searchsorted(spin_start_met, query_met_times, side="right") - 1 ) - # Find all spin time that are less or equal to query_met_times. - # To do that, use side right, a[i-1] <= v < a[i], in the searchsorted. - # Eg. - # >>> df['a'] - # array([0, 15, 30, 45, 60]) - # >>> np.searchsorted(df['a'], [0, 13, 15, 32, 70], side='right') - # array([1, 1, 2, 3, 5]) - last_spin_indices = ( - np.searchsorted(spin_start_met, query_met_times, side="right") - 1 - ) - - # Generate a dataframe with one row per query time - out_df = spin_df.iloc[last_spin_indices].copy() - - # Get the precomputed actual spin period for each query time - spin_periods_for_query = actual_spin_periods[last_spin_indices] - - # Calculate spin phase using actual computed periods - spin_phases = ( - query_met_times - out_df["spin_start_met"].values - ) / spin_periods_for_query - - # Check for invalid spin phase using below checks: - # 1. Check that the spin phase is in valid range, [0, 1). - # 2. Check invalid spin phase using spin_phase_valid, - # spin_period_valid columns. - invalid_spin_phase_range = (spin_phases < 0) | (spin_phases >= 1) - - # TODO: add optional to filter this if this flag means - # that repointing is happening. otherwise, then keep it. - # This needs to be discussed and receive guidance at - # the project level. - invalid_spins = (out_df["spin_phase_valid"].values == 0) | ( - out_df["spin_period_valid"].values == 0 - ) - bad_spin_phases = invalid_spin_phase_range | invalid_spins - spin_phases[bad_spin_phases] = np.nan - - # Add spin_phase column to output dataframe - out_df["sc_spin_phase"] = spin_phases + # Calculate spin phase using actual computed periods + spin_phases = ( + query_met_times - spin_start_met[last_spin_indices] + ) / actual_spin_periods[last_spin_indices] + + # Check for invalid spin phase using below checks: + # 1. Check that the spin phase is in valid range, [0, 1). + # 2. Check invalid spin phase using spin_phase_valid, + # spin_period_valid columns. + invalid_spin_phase_range = (spin_phases < 0) | (spin_phases >= 1) + + # TODO: add optional to filter this if this flag means + # that repointing is happening. otherwise, then keep it. + # This needs to be discussed and receive guidance at + # the project level. + invalid_spins = (spin_df["spin_phase_valid"].values[last_spin_indices] == 0) | ( + spin_df["spin_period_valid"].values[last_spin_indices] == 0 + ) + bad_spin_phases = invalid_spin_phase_range | invalid_spins + spin_phases[bad_spin_phases] = np.nan + + if suffix == "": + # Generate a dataframe with one row per query time + out_df = spin_df.iloc[last_spin_indices].copy() + # Add spin_number and spin_phase columns to output dataframe + # (spin_number is unchanged for the original spin start times) + out_df[f"spin_number{suffix}"] = spin_df["spin_number"].values[ + last_spin_indices + ] + out_df[f"sc_spin_phase{suffix}"] = spin_phases return out_df @@ -319,8 +310,9 @@ def get_spin_number( spin_number : int or np.ndarray Spin number for the input query time. """ - spin_df = interpolate_spin_data(met_time, use_corrected_spin_start) - spin_numbers = spin_df["spin_number"].values + suffix = "_corr" if use_corrected_spin_start else "" + spin_df = interpolate_spin_data(met_time) + spin_numbers = spin_df[f"spin_number{suffix}"].values return spin_numbers.item() if np.asarray(met_time).ndim == 0 else spin_numbers @@ -384,10 +376,11 @@ def get_spacecraft_spin_phase( spin_phase : float or np.ndarray Spin phase for the input query times. """ - spin_df = interpolate_spin_data(query_met_times, use_corrected_spin_start) + suffix = "_corr" if use_corrected_spin_start else "" + spin_df = interpolate_spin_data(query_met_times) if np.asarray(query_met_times).ndim == 0: - return spin_df["sc_spin_phase"].values[0] - return spin_df["sc_spin_phase"].values + return spin_df[f"sc_spin_phase{suffix}"].values[0] + return spin_df[f"sc_spin_phase{suffix}"].values def get_instrument_spin_phase( diff --git a/imap_processing/tests/spice/test_spin.py b/imap_processing/tests/spice/test_spin.py index 80c84e7e05..723f71d505 100644 --- a/imap_processing/tests/spice/test_spin.py +++ b/imap_processing/tests/spice/test_spin.py @@ -54,7 +54,11 @@ def test_set_spin_table_paths(monkeypatch): False, 15.0, 15.0, # actual_spin_period - 0.0, + 15.0, # spin_start_met_corr + 15.0, # actual_spin_period_corr + 0.0, # sc_spin_phase + 1, # spin_number_corr + 0.0, # sc_spin_phase_corr ] ], ), # Scalar test at spin start time @@ -76,7 +80,11 @@ def test_set_spin_table_paths(monkeypatch): False, 15.0, 15.0, # actual_spin_period - 0.1 / 15, + 15.0, # spin_start_met_corr + 15.0, # actual_spin_period_corr + 0.1 / 15, # sc_spin_phase + 1, # spin_number_corr + 0.1 / 15, # sc_spin_phase_corr ], [ 2, @@ -93,7 +101,11 @@ def test_set_spin_table_paths(monkeypatch): False, 30.0, 15.0, # actual_spin_period - 0.2 / 15, + 30.0, # spin_start_met_corr + 15.0, # actual_spin_period_corr + 0.2 / 15, # sc_spin_phase + 2, # spin_number_corr + 0.2 / 15, # sc_spin_phase_corr ], ], ), # Array test @@ -239,6 +251,8 @@ def test_get_spin_data(use_fake_spin_data_for_time): "spin_start_sec_sclk_corr", "spin_start_subsec_sclk_corr", "spin_start_utc_corr", + "spin_start_met_corr", + "actual_spin_period_corr", }, "Spin data must have the specified fields." From 53093a0e71f8fa33d7b1731cda3fd1e67dfcaf17 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 8 Oct 2026 09:34:13 -0400 Subject: [PATCH 4/5] use corrected spin start times by default in get_spin_data and interpolate_spin_data; overridable --- imap_processing/spice/spin.py | 78 ++++++++++++++++-------- imap_processing/tests/spice/test_spin.py | 6 +- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/imap_processing/spice/spin.py b/imap_processing/spice/spin.py index 383263a828..d6db78da2e 100644 --- a/imap_processing/spice/spin.py +++ b/imap_processing/spice/spin.py @@ -37,13 +37,22 @@ def set_global_spin_table_paths(paths: list[Path]) -> None: config._spin_table_paths = paths -def get_spin_data() -> pd.DataFrame: +def get_spin_data(use_corrected_spin_start: bool = True) -> pd.DataFrame: """ Read spin-tables and return spin data. The spin-tables to read are stored in the mutable module level attribute named `spin_table_paths`. + Parameters + ---------- + use_corrected_spin_start : bool + If True (default), the corrected spin start times replace the original ones: the + values of each `*_corr` column below are returned under the column name + without the `_corr` suffix, and no `_corr` columns are returned. If False, all + the columns below are returned, with the un-suffixed columns computed from the + original spin start times. + Returns ------- spin_data : pandas.DataFrame @@ -83,11 +92,15 @@ def get_spin_data() -> pd.DataFrame: "module attribute spin_table_paths." ) - return _load_spin_data_with_cache(tuple(config._spin_table_paths)) + return _load_spin_data_with_cache( + tuple(config._spin_table_paths), use_corrected_spin_start + ) @functools.cache -def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: +def _load_spin_data_with_cache( + csv_paths: tuple[Path], use_corrected_spin_start: bool +) -> pd.DataFrame: """ Load spin-table data from csv files and combine them. @@ -95,6 +108,9 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: ---------- csv_paths : tuple[Path] Locations of spin-table csv files. + use_corrected_spin_start : bool + If True, the values of each `*_corr` column replace those of the column + without the `_corr` suffix, and the `_corr` columns are dropped. Returns ------- @@ -182,10 +198,18 @@ def _load_spin_data_with_cache(csv_paths: tuple[Path]) -> pd.DataFrame: combined_df[f"actual_spin_period{suffix}"] = np.append( actual_spin_periods, spin_periods_sec[-1] ) + + if use_corrected_spin_start: + corr_columns = [c for c in combined_df.columns if c.endswith("_corr")] + for column in corr_columns: + combined_df[column.removesuffix("_corr")] = combined_df[column] + combined_df = combined_df.drop(columns=corr_columns) return combined_df -def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: +def interpolate_spin_data( + query_met_times: float | npt.NDArray, use_corrected_spin_start: bool = True +) -> pd.DataFrame: """ Interpolate spin table data to the queried MET times. @@ -194,26 +218,30 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: spin phase at the queried MET times. Note that spin phase is by definition, in the interval [0, 1) where 1 is equivalent to 360 degrees. - Rows are selected, and `sc_spin_phase` computed, using the original spin - start times. The `spin_number_corr` and `sc_spin_phase_corr` columns give - the spin number and spin phase computed using the corrected spin start times - instead; these are what :py:func:`get_spin_number` and - :py:func:`get_spacecraft_spin_phase` return. - Parameters ---------- query_met_times : float or np.ndarray Query times in Mission Elapsed Time (MET). + use_corrected_spin_start : bool + If True (default), rows are selected, and `sc_spin_phase` computed, using the + corrected spin start times, and the columns are those of + :py:func:`get_spin_data` with corrected spin start times replacing the + original ones. If False, rows are selected, and + `sc_spin_phase` computed, using the original spin start times; the + `spin_number_corr` and `sc_spin_phase_corr` columns are also added, + giving the spin number and spin phase computed using the corrected spin + start times. Returns ------- spin_df : pandas.DataFrame Spin table data interpolated for each queried MET time. In addition to - the columns output from :py:func:`get_spin_data`, the `sc_spin_phase`, - `spin_number_corr` and `sc_spin_phase_corr` columns are added and are - uniquely computed for each queried MET time. + the columns output from :py:func:`get_spin_data`, the `sc_spin_phase` + column (and the `spin_number_corr` and `sc_spin_phase_corr` columns if + `use_corrected_spin_start` is False) is added and is uniquely computed + for each queried MET time. """ - spin_df = get_spin_data() + spin_df = get_spin_data(use_corrected_spin_start) # Ensure query_met_times is an array query_met_times = np.asarray(query_met_times) @@ -223,9 +251,11 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: # convert scalar to array query_met_times = np.atleast_1d(query_met_times) - # Compute the spin number and spin phase using the original spin start - # times, then using the corrected ones. - for suffix in ("", "_corr"): + # Compute the spin number and spin phase using the un-suffixed spin start + # times (corrected or original per use_corrected_spin_start), then, if the + # original ones were used, using the corrected ones. + suffixes = ("",) if use_corrected_spin_start else ("", "_corr") + for suffix in suffixes: # Cache frequently accessed arrays to avoid repeated .values calls spin_start_met = spin_df[f"spin_start_met{suffix}"].values actual_spin_periods = spin_df[f"actual_spin_period{suffix}"].values @@ -277,7 +307,7 @@ def interpolate_spin_data(query_met_times: float | npt.NDArray) -> pd.DataFrame: # Generate a dataframe with one row per query time out_df = spin_df.iloc[last_spin_indices].copy() # Add spin_number and spin_phase columns to output dataframe - # (spin_number is unchanged for the original spin start times) + # (spin_number is unchanged for the un-suffixed spin start times) out_df[f"spin_number{suffix}"] = spin_df["spin_number"].values[ last_spin_indices ] @@ -310,9 +340,8 @@ def get_spin_number( spin_number : int or np.ndarray Spin number for the input query time. """ - suffix = "_corr" if use_corrected_spin_start else "" - spin_df = interpolate_spin_data(met_time) - spin_numbers = spin_df[f"spin_number{suffix}"].values + spin_df = interpolate_spin_data(met_time, use_corrected_spin_start) + spin_numbers = spin_df["spin_number"].values return spin_numbers.item() if np.asarray(met_time).ndim == 0 else spin_numbers @@ -376,11 +405,10 @@ def get_spacecraft_spin_phase( spin_phase : float or np.ndarray Spin phase for the input query times. """ - suffix = "_corr" if use_corrected_spin_start else "" - spin_df = interpolate_spin_data(query_met_times) + spin_df = interpolate_spin_data(query_met_times, use_corrected_spin_start) if np.asarray(query_met_times).ndim == 0: - return spin_df[f"sc_spin_phase{suffix}"].values[0] - return spin_df[f"sc_spin_phase{suffix}"].values + return spin_df["sc_spin_phase"].values[0] + return spin_df["sc_spin_phase"].values def get_instrument_spin_phase( diff --git a/imap_processing/tests/spice/test_spin.py b/imap_processing/tests/spice/test_spin.py index 723f71d505..eab30533e4 100644 --- a/imap_processing/tests/spice/test_spin.py +++ b/imap_processing/tests/spice/test_spin.py @@ -114,7 +114,9 @@ def test_set_spin_table_paths(monkeypatch): def test_interpolate_spin_data(query_met_times, expected, fake_spin_data): """Test interpolate_spin_data() with generated spin data.""" # Call the function - spin_df = spin.interpolate_spin_data(query_met_times=query_met_times) + spin_df = spin.interpolate_spin_data( + query_met_times=query_met_times, use_corrected_spin_start=False + ) # Test the value for i_row, row in enumerate(expected): @@ -228,7 +230,7 @@ def test_get_spacecraft_spin_phase_value_error(query_met_times, fake_spin_data): def test_get_spin_data(use_fake_spin_data_for_time): """Test get_spin_data() with generated spin data.""" use_fake_spin_data_for_time(453051323.0 - 56120) - spin_data = spin.get_spin_data() + spin_data = spin.get_spin_data(use_corrected_spin_start=False) ( np.testing.assert_array_equal(spin_data.index, np.arange(5761)), From 0c750f85d1378b2dadeee4011f27b3d9f3e6146f Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 8 Oct 2026 11:45:56 -0400 Subject: [PATCH 5/5] GLOWS is using uncorrected data from spin.py --- imap_processing/glows/l1b/glows_l1b_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 7165e35662..c2dcd4d246 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -995,7 +995,7 @@ def update_spice_parameters(self, spin_offset_correction: float = 0.0) -> None: # Calculate spin period # --------------------- - spin_data = get_spin_data() + spin_data = get_spin_data(use_corrected_spin_start=False) # select spin data within the range from data start time to end time spin_data = spin_data[ (spin_data["spin_start_met"] >= data_start_met) @@ -1009,7 +1009,9 @@ def update_spice_parameters(self, spin_offset_correction: float = 0.0) -> None: # -------------------------------- angle_offset = 360 - get_spin_angle( get_instrument_spin_phase( - self.imap_start_time, instrument=geometry.SpiceFrame.IMAP_GLOWS + self.imap_start_time, + instrument=geometry.SpiceFrame.IMAP_GLOWS, + use_corrected_spin_start=False, ), degrees=True, )