From 7d442c256b232bd5f94ecfc6cfa909886b330d2e Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 00:04:31 +0200 Subject: [PATCH 1/8] Add the root eocontainer marker and drop duplicated stac_discovery keys Co-Authored-By: Claude Opus 5.5 --- docs/DATATREE.md | 18 ++++++++++++------ tests/data/eopf/S01SIWSLC_target_attrs.json | 3 --- tests/test_30_eopf_product.py | 3 +++ xarray_sentinel/eopf_metadata.py | 8 -------- xarray_sentinel/eopf_product.py | 2 ++ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/DATATREE.md b/docs/DATATREE.md index 12d2161..7c4d512 100644 --- a/docs/DATATREE.md +++ b/docs/DATATREE.md @@ -104,11 +104,11 @@ All fields come from the SAFE name, except `burst_id` from `swathTiming` and ### Attributes - root - - `stac_discovery`: STAC item from `manifest.safe`, one asset per node. - - `other_metadata`: `{"eopf_category": "eocontainer", "title": "S01SIWSLC"}`. + - `other_metadata`: `{"eopf_category": "eocontainer", "title": "S01SIWSLC"}`. The CPM + also has a root `stac_discovery`, we don't, see step 9. - node - `stac_discovery`: manifest and annotation properties, `sar:relative_burst`, burst - footprint as `geometry`, `detectionPerformed`, `thermalNoiseCorrectionPerformed`. + footprint as `geometry`. - `processing_history`: from the manifest `safe:processing`. - `other_metadata`: keyed by polarisation, each with `eopf_category: "eoproduct"`, `title`, `image_number`, `quality_information`, `general_annotation`, @@ -182,8 +182,10 @@ What we do in each case follows the decisions above. - `sar:relative_burst` needs the SAR extension v1.3.1, the CPM declares v1.3.0. - the eopf extension URL does not exist, we use `https://cs-si.github.io/eopf-stac-extension/v1.2.0/schema.json`. - - `detectionPerformed` and `thermalNoiseCorrectionPerformed` are strings, we use - bools, `links` points to a Zarr v2 `.zattrs.json`, we use none. + - `detectionPerformed` and `thermalNoiseCorrectionPerformed` are non-STAC top-level + keys, already in `other_metadata`, we drop them. `links` points to a Zarr v2 + `.zattrs.json`, we use none. + - `eopf:instrument_mode` duplicates `sar:instrument_mode`, we drop it. ## Open points @@ -268,7 +270,11 @@ adds converters to the step 1 registry, steps 10 and 11 come last. `stac_discovery_annotation_manifest`, quirk 8. `geometry` is the node footprint interpolated from the GCPs, `sar:relative_burst` is set when the product has burst ids. The items validate against the declared STAC extensions. - 1. root `stac_discovery` and `other_metadata`. + 1. Done: root `other_metadata`, `{eopf_category: "eocontainer", title}` with + `eopf_category` used by the CPM tools to recognise the container. No root + `stac_discovery`: the CPM reads it from the manifest and merges its properties + into every node, then adds assets from the variable short names, which collide + across nodes, and its own software versions. 1. node `other_metadata` extending `eopf_metadata.build_other_metadata`: `eopf_category`, `title`, `image_number`, `burst_timing` with the node's burst merged in, drop empty `swath_merging`, numeric lists instead of `$` strings, diff --git a/tests/data/eopf/S01SIWSLC_target_attrs.json b/tests/data/eopf/S01SIWSLC_target_attrs.json index bd61a7f..670b946 100644 --- a/tests/data/eopf/S01SIWSLC_target_attrs.json +++ b/tests/data/eopf/S01SIWSLC_target_attrs.json @@ -2,7 +2,6 @@ "stac_discovery": { "assets": {}, "bbox": "list", - "detectionPerformed": "bool", "geometry": { "coordinates": "list", "type": "str" @@ -15,7 +14,6 @@ "datetime": "str", "end_datetime": "str", "eopf:datatake_id": "str", - "eopf:instrument_mode": "str", "instruments": "list", "mission": "str", "platform": "str", @@ -49,7 +47,6 @@ }, "stac_extensions": "list", "stac_version": "str", - "thermalNoiseCorrectionPerformed": "bool", "type": "str" } } diff --git a/tests/test_30_eopf_product.py b/tests/test_30_eopf_product.py index 0e279b3..e0c126b 100644 --- a/tests/test_30_eopf_product.py +++ b/tests/test_30_eopf_product.py @@ -149,6 +149,9 @@ def test_open_datatree_attrs() -> None: node = res.children["S01SIWSLC_20220414T102209_0027_A244_E677_051AA4_IW1_365915"] stac_discovery = node.attrs["stac_discovery"] + assert res.attrs == { + "other_metadata": {"eopf_category": "eocontainer", "title": "S01SIWSLC"} + } assert { key: attrs_structure(node.attrs[key]) for key in EXPECTED_SLC_IW_ATTRS } == EXPECTED_SLC_IW_ATTRS diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index 6687b74..08f0878 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -143,9 +143,6 @@ def build_stac_discovery( annotation, "generalAnnotation/productInformation" ) image_information = esa_safe.get_tag(annotation, "imageAnnotation/imageInformation") - processing_information = esa_safe.get_tag( - annotation, "imageAnnotation/processingInformation" - ) if footprint is None: # the manifest footprint is in lat,lon order @@ -215,7 +212,6 @@ def build_stac_discovery( manifest, ".//s1sarl1:instrumentMode/s1sarl1:swath" ), "eopf:datatake_id": esa_safe.findtext(manifest, ".//s1sarl1:missionDataTakeID"), - "eopf:instrument_mode": mode, } if relative_burst is not None: properties["sar:relative_burst"] = relative_burst @@ -233,9 +229,5 @@ def build_stac_discovery( "properties": properties, "links": [], "assets": {}, - "detectionPerformed": processing_information["detectionPerformed"], - "thermalNoiseCorrectionPerformed": processing_information[ - "thermalNoiseCorrectionPerformed" - ], } return stac_item diff --git a/xarray_sentinel/eopf_product.py b/xarray_sentinel/eopf_product.py index 1f1eef7..867c880 100644 --- a/xarray_sentinel/eopf_product.py +++ b/xarray_sentinel/eopf_product.py @@ -196,4 +196,6 @@ def open_datatree( for eopf_group, ds in node_groups.items(): dt[f"{node}/{eopf_group}"] = ds + title = f"S01S{common_attrs['mode']}{common_attrs['product_type']}" + dt.attrs = {"other_metadata": {"eopf_category": "eocontainer", "title": title}} return dt From 8c7d8b3ebb552e3b45b92943ae6ebfd685586af5 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 08:27:36 +0200 Subject: [PATCH 2/8] Convert Sentinel-1 arrays to float lists and empty lists to [] in other_metadata Co-Authored-By: Claude Opus 5.5 --- tests/test_15_eopf_metadata.py | 13 ++++++++++ xarray_sentinel/eopf_metadata.py | 44 ++++++++++++++------------------ 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/test_15_eopf_metadata.py b/tests/test_15_eopf_metadata.py index 997a0cf..40eb410 100644 --- a/tests/test_15_eopf_metadata.py +++ b/tests/test_15_eopf_metadata.py @@ -118,6 +118,19 @@ def test_build_other_metadata() -> None: assert res["quality_information"] == expected_quality_information + general_annotation = res["general_annotation"] + replica = general_annotation["replica_information_list"][0]["reference_replica"] + fm_rate = general_annotation["azimuth_fm_rate_list"][0] + + assert all(isinstance(c, float) for c in replica["amplitude_coefficients"]) + assert all(isinstance(c, float) for c in fm_rate["azimuth_fm_rate_polynomial"]) + assert res["swath_timing"]["burst_list"] == [] + + res = eopf_metadata.build_other_metadata(esa_safe.parse_xml(SLC_IW1_VV_annotation)) + burst = res["swath_timing"]["burst_list"][0] + + assert all(isinstance(s, float) for s in burst["first_valid_sample"]) + @pytest.mark.parametrize( "safe,annotation", diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index 08f0878..346b581 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -45,17 +45,31 @@ def to_snake_recursive( def fix_lists(struct: Any) -> Any: + """Simplify the xmlschema decoding of a Sentinel-1 XML to plain values. + + xmlschema writes attributes as "@name" keys and the text of an element with + attributes under "$". On top of that the Sentinel-1 schemas use two patterns: + arrays (floatArray, ... in s1-object-types.xsd) are strings of space separated + numbers with a count attribute, {"@count": 3, "$": "1 2 3"}, and lists like + burst_list hold the burst elements and a count, {"@count": 2, "burst": [...]}. + + Drop the count attributes, replace an element with attributes by its value, an + array by a list of floats and a list by its items. + """ fixed: Any if isinstance(struct, dict): fixed = {} for k, v in struct.items(): if k == "@count": continue - if k[-5:] == "_list": - try: - fixed[k] = fix_lists(v[k[:-5]]) - except Exception: - fixed[k] = fix_lists(fix_lists(v)) + # an element with attributes or an array: the dict becomes its value + elif k == "$": + values = ( + [float(value) for value in v.split()] if isinstance(v, str) else v + ) + return values + elif k[-5:] == "_list": + fixed[k] = fix_lists(v.get(k[:-5], [])) else: fixed[k] = fix_lists(v) elif isinstance(struct, list): @@ -71,30 +85,10 @@ def filter_metadata_dict(image_information: dict[str, Any]) -> dict[str, Any]: return image_information -def build_azimuth_fm_rate_list( - azimuth_fm_rate_list: list[dict[str, Any] | list[Any]], -) -> list[dict[str, Any] | list[Any]]: - azimuth_fm_rate_list_out: list[dict[str, Any] | list[Any]] = [] - for item in azimuth_fm_rate_list: - azimuth_fm_rate_polynomial_str = item["azimuth_fm_rate_polynomial"]["$"] # type: ignore - azimuth_fm_rate_list_out.append( - { - "azimuth_time": item["azimuth_time"], # type: ignore - "azimuth_fm_rate_polynomial": np.fromstring( - azimuth_fm_rate_polynomial_str, sep=" " - ).tolist(), - } - ) - return azimuth_fm_rate_list_out - - def build_general_annotation(general_annotation: dict[str, Any]) -> dict[str, Any]: general_annotation = filter_metadata_dict(general_annotation) _ = general_annotation.pop("orbit_list") _ = general_annotation.pop("attitude_list") - general_annotation["azimuth_fm_rate_list"] = build_azimuth_fm_rate_list( - general_annotation["azimuth_fm_rate_list"] - ) return general_annotation From b8139ebfaeec2ba56624391972e62658dfe67c6a Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 08:40:15 +0200 Subject: [PATCH 3/8] Build the node other_metadata per burst like the CPM Co-Authored-By: Claude Opus 5.5 --- docs/DATATREE.md | 8 +++-- tests/data/eopf/S01SIWSLC_target_attrs.json | 29 ++++++++++++++++++ tests/test_15_eopf_metadata.py | 33 ++++++++++++++------ tests/test_30_eopf_product.py | 22 ++++++++----- xarray_sentinel/eopf_metadata.py | 34 +++++++++++++++++---- xarray_sentinel/eopf_product.py | 14 ++++++--- 6 files changed, 111 insertions(+), 29 deletions(-) diff --git a/docs/DATATREE.md b/docs/DATATREE.md index 7c4d512..f65c556 100644 --- a/docs/DATATREE.md +++ b/docs/DATATREE.md @@ -264,7 +264,8 @@ adds converters to the step 1 registry, steps 10 and 11 come last. xarray-sentinel finds one annotation per WV swath instead of one per vignette. 1. Attributes, in four parts. `tests/data/eopf/S01SIWSLC_target_attrs.json` holds the key and type tree of the node attributes, the CPM 3.0 burst node with our deliberate - differences. + differences, with `other_metadata` of one polarisation and its annotation sections as + `dict` or `list` only, as their content depends on the IPF version. 1. Done: node `stac_discovery`, `eopf_metadata.build_stac_discovery` following the CPM mapping keys `stac_discovery`, `stac_discovery_annotation` and `stac_discovery_annotation_manifest`, quirk 8. `geometry` is the node footprint @@ -275,10 +276,11 @@ adds converters to the step 1 registry, steps 10 and 11 come last. `stac_discovery`: the CPM reads it from the manifest and merges its properties into every node, then adds assets from the variable short names, which collide across nodes, and its own software versions. - 1. node `other_metadata` extending `eopf_metadata.build_other_metadata`: + 1. Done: node `other_metadata`, `eopf_metadata.build_other_metadata` with `eopf_category`, `title`, `image_number`, `burst_timing` with the node's burst merged in, drop empty `swath_merging`, numeric lists instead of `$` strings, - `t_0` as `t0`. We keep the CPM `DEPRECATED_ATTRS`. + `t_0` as `t0`. `title` is the product type from the manifest, as the annotation + mode of SM is the beam. We keep the CPM `DEPRECATED_ATTRS`. 1. `processing_history`, by level from the manifest as in CPM 2.7, the CPM 3.0 one is a single `"Unknown"` entry. 1. API and docs: `Sentinel1Backend.open_datatree` and `open_groups_as_dict`, README, diff --git a/tests/data/eopf/S01SIWSLC_target_attrs.json b/tests/data/eopf/S01SIWSLC_target_attrs.json index 670b946..3046cfe 100644 --- a/tests/data/eopf/S01SIWSLC_target_attrs.json +++ b/tests/data/eopf/S01SIWSLC_target_attrs.json @@ -1,4 +1,33 @@ { + "other_metadata": { + "burst_timing": { + "azimuth_anx_time": "float", + "azimuth_time": "str", + "burst_id": "int", + "first_valid_sample": "list", + "last_valid_sample": "list", + "lines_per_burst": "int", + "samples_per_burst": "int", + "sensing_time": "str" + }, + "eopf_category": "str", + "general_annotation": { + "azimuth_fm_rate_list": "list", + "downlink_information_list": "list", + "noise_list": "list", + "product_information": "dict", + "raw_data_analysis_list": "list", + "replica_information_list": "list", + "terrain_height_list": "list" + }, + "image_annotation": { + "image_information": "dict", + "processing_information": "dict" + }, + "image_number": "str", + "quality_information": "dict", + "title": "str" + }, "stac_discovery": { "assets": {}, "bbox": "list", diff --git a/tests/test_15_eopf_metadata.py b/tests/test_15_eopf_metadata.py index 40eb410..9bff11d 100644 --- a/tests/test_15_eopf_metadata.py +++ b/tests/test_15_eopf_metadata.py @@ -37,8 +37,14 @@ def test_to_snake_recursive() -> None: - metadata = {"qualityInformation": {"qualityDataList": [{"qualityData": 2}]}} - expected = {"quality_information": {"quality_data_list": [{"quality_data": 2}]}} + metadata = { + "qualityInformation": {"qualityDataList": [{"qualityData": 2}]}, + "t0": 1, + } + expected = { + "quality_information": {"quality_data_list": [{"quality_data": 2}]}, + "t0": 1, + } res = eopf_metadata.to_snake_recursive(metadata) @@ -46,7 +52,9 @@ def test_to_snake_recursive() -> None: def test_build_other_metadata() -> None: - res = eopf_metadata.build_other_metadata(esa_safe.parse_xml(SLC_S3_VH_annotation)) + res = eopf_metadata.build_other_metadata( + esa_safe.parse_xml(SLC_S3_VH_annotation), "S01SSMSLC" + ) expected_quality_information = { "product_quality_index": 0.0, "quality_data_list": [ @@ -109,11 +117,13 @@ def test_build_other_metadata() -> None: } assert set(res) == { + "eopf_category", + "title", + "image_number", "quality_information", "general_annotation", "image_annotation", - "swath_timing", - "swath_merging", + "burst_timing", } assert res["quality_information"] == expected_quality_information @@ -124,12 +134,17 @@ def test_build_other_metadata() -> None: assert all(isinstance(c, float) for c in replica["amplitude_coefficients"]) assert all(isinstance(c, float) for c in fm_rate["azimuth_fm_rate_polynomial"]) - assert res["swath_timing"]["burst_list"] == [] + assert "t0" in fm_rate + assert "burst_list" not in res["burst_timing"] - res = eopf_metadata.build_other_metadata(esa_safe.parse_xml(SLC_IW1_VV_annotation)) - burst = res["swath_timing"]["burst_list"][0] + res = eopf_metadata.build_other_metadata( + esa_safe.parse_xml(SLC_IW1_VV_annotation), "S01SIWSLC", burst_index=1 + ) + burst_timing = res["burst_timing"] - assert all(isinstance(s, float) for s in burst["first_valid_sample"]) + assert all(isinstance(s, float) for s in burst_timing["first_valid_sample"]) + assert "byte_offset" not in burst_timing + assert burst_timing["azimuth_time"] == "2021-04-01T05:26:26.966491" @pytest.mark.parametrize( diff --git a/tests/test_30_eopf_product.py b/tests/test_30_eopf_product.py index e0c126b..72a04f1 100644 --- a/tests/test_30_eopf_product.py +++ b/tests/test_30_eopf_product.py @@ -45,10 +45,13 @@ def datatree_structure(dt: xr.DataTree) -> dict[str, dict[str, list[Any]]]: } -def attrs_structure(value: Any) -> Any: - if isinstance(value, dict): - return {key: attrs_structure(item) for key, item in value.items()} - return type(value).__name__ +def attrs_structure(value: Any, target: Any) -> Any: + if not (isinstance(value, dict) and isinstance(target, dict)): + return type(value).__name__ + structure = {} + for key, item in value.items(): + structure[key] = attrs_structure(item, target.get(key)) + return structure @pytest.mark.parametrize( @@ -152,8 +155,13 @@ def test_open_datatree_attrs() -> None: assert res.attrs == { "other_metadata": {"eopf_category": "eocontainer", "title": "S01SIWSLC"} } - assert { - key: attrs_structure(node.attrs[key]) for key in EXPECTED_SLC_IW_ATTRS - } == EXPECTED_SLC_IW_ATTRS + assert list(node.attrs["other_metadata"]) == ["HH"] + + # other_metadata of one polarisation + node_attrs = dict(node.attrs) | { + "other_metadata": node.attrs["other_metadata"]["HH"] + } + + assert attrs_structure(node_attrs, EXPECTED_SLC_IW_ATTRS) == EXPECTED_SLC_IW_ATTRS assert stac_discovery["id"] == node.name assert stac_discovery["properties"]["sar:relative_burst"] == 365915 diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index 346b581..ea59c28 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -34,11 +34,19 @@ def to_snake(camel: str) -> str: return snake.lower() +# to_snake splits a letter followed by a digit, the EOPF names keep them together +SNAKE_CASE_FIXES = {"t_0": "t0"} + + def to_snake_recursive( struct: dict[str, Any] | list[Any], ) -> dict[str, Any] | list[Any]: if isinstance(struct, dict): - struct = {to_snake(k): to_snake_recursive(v) for k, v in struct.items()} + snake_struct = {} + for k, v in struct.items(): + key = to_snake(k) + snake_struct[SNAKE_CASE_FIXES.get(key, key)] = to_snake_recursive(v) + struct = snake_struct elif isinstance(struct, list): struct = [to_snake_recursive(v) for v in struct] return struct @@ -92,21 +100,35 @@ def build_general_annotation(general_annotation: dict[str, Any]) -> dict[str, An return general_annotation -def build_other_metadata(annotation: dict[str, Any]) -> dict[str, Any]: +def make_product_type(mode: str, product_type: str) -> str: + return f"S01S{mode}{product_type}" + + +def build_other_metadata( + annotation: dict[str, Any], title: str, burst_index: int | None = None +) -> dict[str, Any]: warnings.warn("This is an unofficial, alpha converter", UserWarning) quality_information = filter_metadata_dict(annotation["qualityInformation"]) general_annotation = build_general_annotation(annotation["generalAnnotation"]) image_information = filter_metadata_dict(annotation["imageAnnotation"]) swath_merging = filter_metadata_dict(annotation["swathMerging"]) - swath_timing = filter_metadata_dict(annotation["swathTiming"]) + burst_timing = filter_metadata_dict(annotation["swathTiming"]) + bursts = burst_timing.pop("burst_list") + if burst_index is not None: + burst_timing |= bursts[burst_index] + _ = burst_timing.pop("byte_offset") other_metadata = { + "eopf_category": "eoproduct", + "title": title, + "image_number": annotation["adsHeader"]["imageNumber"], "quality_information": quality_information, "general_annotation": general_annotation, "image_annotation": image_information, - "swath_timing": swath_timing, - "swath_merging": swath_merging, + "burst_timing": burst_timing, } + if swath_merging["swath_merge_list"]: + other_metadata["swath_merging"] = swath_merging return other_metadata @@ -174,7 +196,7 @@ def build_stac_discovery( {"name": facility.attrib["organisation"], "roles": ["producer"]}, ], "constellation": "sentinel-1", - "product:type": f"S01S{mode}{product_type}", + "product:type": make_product_type(mode, product_type), "product:timeliness_category": timeliness_category, "product:timeliness": PRODUCT_TIMELINESS[timeliness_category], "processing:datetime": processing.attrib["stop"] + "Z", diff --git a/xarray_sentinel/eopf_product.py b/xarray_sentinel/eopf_product.py index 867c880..84b5d49 100644 --- a/xarray_sentinel/eopf_product.py +++ b/xarray_sentinel/eopf_product.py @@ -128,12 +128,14 @@ def open_datatree( swath_pols.setdefault(swath, []).append(pol) product_name = make_product_name(pathlib.PurePosixPath(product_path).stem) + title = eopf_metadata.make_product_type( + common_attrs["mode"], common_attrs["product_type"] + ) dt = xr.DataTree() for swath, pols in swath_pols.items(): bursts: dict[str | None, dict[str, xr.Dataset]] = {} pol_datasets: dict[str, dict[str, xr.Dataset]] = {} annotations = {} - other_metadata = {} for pol in pols: measurement_path, annotation_path = groups[f"{swath}/{pol}"][:2] annotations[pol] = esa_safe.parse_xml(xml_files[annotation_path]) @@ -160,7 +162,6 @@ def open_datatree( converted = converter(annotations[pol]) if converted is not None: pol_datasets.setdefault(eopf_group, {})[pol] = converted - other_metadata[pol] = eopf_metadata.build_other_metadata(annotations[pol]) annotation = annotations[pols[0]] burst_list = esa_safe.get_tag(annotation, "swathTiming/burstList") @@ -173,7 +174,7 @@ def open_datatree( else: node_groups[eopf_group] = stack_polarizations(datasets) - for burst_id, burst_pols in bursts.items(): + for burst_index, (burst_id, burst_pols) in enumerate(bursts.items()): node = product_name if burst_id is not None: node += f"_{swath}_{burst_id}" @@ -186,6 +187,12 @@ def open_datatree( stac_discovery = eopf_metadata.build_stac_discovery( node, manifest, annotation, footprint, relative_burst ) + node_burst_index = None if burst_id is None else burst_index + other_metadata = {} + for pol in pols: + other_metadata[pol] = eopf_metadata.build_other_metadata( + annotations[pol], title, node_burst_index + ) dt[node] = xr.Dataset( attrs={ "other_metadata": other_metadata, @@ -196,6 +203,5 @@ def open_datatree( for eopf_group, ds in node_groups.items(): dt[f"{node}/{eopf_group}"] = ds - title = f"S01S{common_attrs['mode']}{common_attrs['product_type']}" dt.attrs = {"other_metadata": {"eopf_category": "eocontainer", "title": title}} return dt From fd2a289d16ac260a1efa3665f7b0e0748b58f11f Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 09:00:25 +0200 Subject: [PATCH 4/8] Add the build_processing_history, but don't connet it for now --- docs/DATATREE.md | 12 ++++++-- tests/test_15_eopf_metadata.py | 18 +++++++++++ xarray_sentinel/eopf_metadata.py | 52 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/DATATREE.md b/docs/DATATREE.md index f65c556..d31fdea 100644 --- a/docs/DATATREE.md +++ b/docs/DATATREE.md @@ -109,7 +109,7 @@ All fields come from the SAFE name, except `burst_id` from `swathTiming` and - node - `stac_discovery`: manifest and annotation properties, `sar:relative_burst`, burst footprint as `geometry`. - - `processing_history`: from the manifest `safe:processing`. + - `processing_history`: from the manifest `safe:processing`, not set for now. - `other_metadata`: keyed by polarisation, each with `eopf_category: "eoproduct"`, `title`, `image_number`, `quality_information`, `general_annotation`, `image_annotation` and `burst_timing`. `burst_timing` holds the node's burst entry of @@ -186,6 +186,10 @@ What we do in each case follows the decisions above. keys, already in `other_metadata`, we drop them. `links` points to a Zarr v2 `.zattrs.json`, we use none. - `eopf:instrument_mode` duplicates `sar:instrument_mode`, we drop it. +1. `processing_history` is a single `"Unknown"` entry with no time, inputs or outputs: + `ToProcessingHistory` tests lxml attributes with `"name" in node`, which checks + the children. We walk the manifest as CPM 2.7 did, without its own conversion entry, + but don't set it yet. ## Open points @@ -281,8 +285,10 @@ adds converters to the step 1 registry, steps 10 and 11 come last. merged in, drop empty `swath_merging`, numeric lists instead of `$` strings, `t_0` as `t0`. `title` is the product type from the manifest, as the annotation mode of SM is the beam. We keep the CPM `DEPRECATED_ATTRS`. - 1. `processing_history`, by level from the manifest as in CPM 2.7, the CPM 3.0 one - is a single `"Unknown"` entry. + 1. Done, not connected: node `processing_history`, + `eopf_metadata.build_processing_history`, quirk 9, entries keyed by output level + from the resource roles and sorted by time, AUX resources as `adfs` and not + walked. `open_datatree` does not set it for now. 1. API and docs: `Sentinel1Backend.open_datatree` and `open_groups_as_dict`, README, this file. 1. Validation: convert one or two full SAFE products with CPM 3.0 in a throwaway diff --git a/tests/test_15_eopf_metadata.py b/tests/test_15_eopf_metadata.py index 9bff11d..39336c1 100644 --- a/tests/test_15_eopf_metadata.py +++ b/tests/test_15_eopf_metadata.py @@ -147,6 +147,24 @@ def test_build_other_metadata() -> None: assert burst_timing["azimuth_time"] == "2021-04-01T05:26:26.966491" +def test_build_processing_history() -> None: + res = eopf_metadata.build_processing_history(SLC_IW / "manifest.safe", SLC_IW.name) + + assert set(res) == { + "Level-0 Product", + "Level-1 Intermediate SLC Product", + "Level-1 Product", + } + (product,) = res["Level-1 Product"] + assert product["outputs"] == [SLC_IW.name] + (intermediate,) = res["Level-1 Intermediate SLC Product"] + assert product["inputs"] == intermediate["outputs"] + assert len(intermediate["adfs"]) == 3 + assert res["Level-0 Product"][-1]["facility"] == "PAC1" + times = [entry["time"] for entry in res["Level-0 Product"]] + assert times == sorted(times) + + @pytest.mark.parametrize( "safe,annotation", [ diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index ea59c28..03acd22 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -247,3 +247,55 @@ def build_stac_discovery( "assets": {}, } return stac_item + + +ROLE_LEVELS = {"Raw Data": "Level-0 Product"} + + +def build_processing_history( + manifest_path: esa_safe.PathOrFileType, + safe_name: str, + namespaces: dict[str, str] = esa_safe.SENTINEL1_NAMESPACES, +) -> dict[str, list[dict[str, Any]]]: + """Group the processing steps of the manifest by output level, as CPM 2.7.""" + if hasattr(manifest_path, "seek"): + manifest_path.seek(0) + manifest = ElementTree.parse(manifest_path).getroot() + + processing_history: dict[str, list[dict[str, Any]]] = {} + processing = manifest.find(".//safe:processing", namespaces) + assert processing is not None + stack = [(processing, "Level-1 Product", safe_name)] + while stack: + processing, level, output = stack.pop() + facility = processing.find("safe:facility", namespaces) + software = processing.find("safe:facility/safe:software", namespaces) + facility_name = "Unknown" + if facility is not None: + parts = [facility.get("organisation"), facility.get("name")] + facility_name = "-".join(part for part in parts if part) + software_attrs = {} if software is None else software.attrib + entry: dict[str, Any] = { + "processor": software_attrs.get("name", "Unknown"), + "version": software_attrs.get("version", "Unknown"), + "facility": facility_name, + "time": processing.attrib["stop"] + "Z", + "inputs": [], + "outputs": [output], + } + adfs = [] + for resource in processing.findall("safe:resource", namespaces): + name, role = resource.attrib["name"], resource.attrib["role"] + if "AUX" in name: + adfs.append(name) + continue + entry["inputs"].append(name) + for child in resource.findall("safe:processing", namespaces): + stack.append((child, ROLE_LEVELS.get(role, role), name)) + if adfs: + entry["adfs"] = adfs + processing_history.setdefault(level, []).append(entry) + + for entries in processing_history.values(): + entries.sort(key=lambda entry: entry["time"]) + return processing_history From 5decb47ae46137dcc64e81b798cf7b706c306c13 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 09:11:34 +0200 Subject: [PATCH 5/8] Simplify the $ value conversion in fix_lists Co-Authored-By: Claude Opus 5.5 --- xarray_sentinel/eopf_metadata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index 03acd22..ff9d93d 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -72,10 +72,9 @@ def fix_lists(struct: Any) -> Any: continue # an element with attributes or an array: the dict becomes its value elif k == "$": - values = ( - [float(value) for value in v.split()] if isinstance(v, str) else v - ) - return values + if isinstance(v, str): + v = [float(value) for value in v.split()] + return v elif k[-5:] == "_list": fixed[k] = fix_lists(v.get(k[:-5], [])) else: From 3dee1163bd2387e6ea8ba34e419d11b0e1335e0f Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 09:16:55 +0200 Subject: [PATCH 6/8] Use key and value as loop names in fix_lists and to_snake_recursive Co-Authored-By: Claude Opus 5.5 --- xarray_sentinel/eopf_metadata.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index ff9d93d..4774791 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -43,9 +43,10 @@ def to_snake_recursive( ) -> dict[str, Any] | list[Any]: if isinstance(struct, dict): snake_struct = {} - for k, v in struct.items(): - key = to_snake(k) - snake_struct[SNAKE_CASE_FIXES.get(key, key)] = to_snake_recursive(v) + for key, value in struct.items(): + snake_key = to_snake(key) + snake_key = SNAKE_CASE_FIXES.get(snake_key, snake_key) + snake_struct[snake_key] = to_snake_recursive(value) struct = snake_struct elif isinstance(struct, list): struct = [to_snake_recursive(v) for v in struct] @@ -67,18 +68,18 @@ def fix_lists(struct: Any) -> Any: fixed: Any if isinstance(struct, dict): fixed = {} - for k, v in struct.items(): - if k == "@count": + for key, value in struct.items(): + if key == "@count": continue # an element with attributes or an array: the dict becomes its value - elif k == "$": - if isinstance(v, str): - v = [float(value) for value in v.split()] - return v - elif k[-5:] == "_list": - fixed[k] = fix_lists(v.get(k[:-5], [])) + elif key == "$": + if isinstance(value, str): + value = [float(v) for v in value.split()] + return value + elif key[-5:] == "_list": + fixed[key] = fix_lists(value.get(key[:-5], [])) else: - fixed[k] = fix_lists(v) + fixed[key] = fix_lists(value) elif isinstance(struct, list): fixed = [fix_lists(v) for v in struct] else: From aecfee689014e7fd830bb65596a8b458def4a85f Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 09:31:42 +0200 Subject: [PATCH 7/8] Drop the numpy and pathlib imports from the EOPF modules Co-Authored-By: Claude Opus 5.5 --- xarray_sentinel/eopf_metadata.py | 15 ++++++++------- xarray_sentinel/eopf_product.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/xarray_sentinel/eopf_metadata.py b/xarray_sentinel/eopf_metadata.py index 4774791..7835aaa 100644 --- a/xarray_sentinel/eopf_metadata.py +++ b/xarray_sentinel/eopf_metadata.py @@ -1,10 +1,9 @@ +import datetime import re import warnings from typing import Any from xml.etree import ElementTree -import numpy as np - from . import esa_safe @@ -169,8 +168,10 @@ def build_stac_discovery( footprint.append(footprint[0]) lons, lats = zip(*footprint) - start_time = np.datetime64(esa_safe.findtext(manifest, ".//safe:startTime"), "us") - stop_time = np.datetime64(esa_safe.findtext(manifest, ".//safe:stopTime"), "us") + start = esa_safe.findtext(manifest, ".//safe:startTime") + stop = esa_safe.findtext(manifest, ".//safe:stopTime") + start_time = datetime.datetime.fromisoformat(start) + stop_time = datetime.datetime.fromisoformat(stop) middle_time = start_time + (stop_time - start_time) // 2 processing = manifest.find(".//safe:processing", namespaces=namespaces) facility = manifest.find(".//safe:processing/safe:facility", namespaces=namespaces) @@ -184,9 +185,9 @@ def build_stac_discovery( properties = { "mission": "copernicus", - "datetime": np.datetime_as_string(middle_time) + "Z", - "start_datetime": np.datetime_as_string(start_time) + "Z", - "end_datetime": np.datetime_as_string(stop_time) + "Z", + "datetime": middle_time.isoformat(timespec="microseconds") + "Z", + "start_datetime": start + "Z", + "end_datetime": stop + "Z", "created": processing.attrib["stop"] + "Z", "platform": "sentinel-1" + esa_safe.findtext(manifest, ".//safe:platform/safe:number").lower(), diff --git a/xarray_sentinel/eopf_product.py b/xarray_sentinel/eopf_product.py index 84b5d49..4b78b26 100644 --- a/xarray_sentinel/eopf_product.py +++ b/xarray_sentinel/eopf_product.py @@ -1,7 +1,6 @@ import datetime import io import os -import pathlib import re from collections.abc import Callable from typing import Any @@ -127,7 +126,8 @@ def open_datatree( swath, _, pol = group.partition("/") swath_pols.setdefault(swath, []).append(pol) - product_name = make_product_name(pathlib.PurePosixPath(product_path).stem) + safe_name, _ = os.path.splitext(os.path.basename(product_path)) + product_name = make_product_name(safe_name) title = eopf_metadata.make_product_type( common_attrs["mode"], common_attrs["product_type"] ) From aeb94f099502e4a742ccb4bd470413ded6792d97 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 18 Oct 2026 09:57:07 +0200 Subject: [PATCH 8/8] Fix the dead ESA links, add the eopf_sentinel1 references and the parse-once step Co-Authored-By: Claude Opus 5.5 --- README.md | 14 +++++++------- docs/DATATREE.md | 15 ++++++++++++++- xarray_sentinel/conventions.py | 2 +- xarray_sentinel/eopf_sentinel1.py | 14 ++++++++++++++ xarray_sentinel/sentinel1.py | 8 ++++---- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ff01ebc..a7c4e4e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # xarray-sentinel Easily access and explore the SAR data products of the -[Copernicus Sentinel-1 satellite mission](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-1) +[Copernicus Sentinel-1 satellite mission](https://sentiwiki.copernicus.eu/web/sentinel-1) in Python. This Open Source project is sponsored by B-Open - . @@ -543,12 +543,12 @@ Attributes: ... This is the list of the reference documents: -- Sentinel-1 document library: - - [user guides](https://sentinels.copernicus.eu/web/sentinel/user-guides/sentinel-1-sar) - - [technical guides](https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-1-sar) -- [Sentinel-1 Product Specification v3.9 07 May 2021 S1-RS-MDA-52-7441-3-9 documenting IPF 3.40](https://sentinel.esa.int/documents/247904/1877131/S1-RS-MDA-52-7441-3-9-2_Sentinel-1ProductSpecification.pdf) -- [Sentinel-1 Product Specification v3.7 27 February 2020 S1-RS-MDA-52-7441 documenting IPF 3.30](https://sentinel.esa.int/documents/247904/1877131/Sentinel-1-Product-Specification) -- [Radiometric Calibration of S-1 Level-1 Products Generated by the S-1 IPF v1.0 21/05/2015 ESA-EOPG-CSCOP-TN-0002](https://sentinel.esa.int/documents/247904/685163/S1-Radiometric-Calibration-V1.0.pdf) +- Sentinel-1 on SentiWiki: + - [user guides](https://sentiwiki.copernicus.eu/web/sentinel-1) + - [documents](https://sentiwiki.copernicus.eu/web/s1-documents) +- [Sentinel-1 Product Specification v3.9 07 May 2021 S1-RS-MDA-52-7441-3-9 documenting IPF 3.40](https://sentinels.copernicus.eu/documents/247904/1877131/S1-RS-MDA-52-7441-3-9-2_Sentinel-1ProductSpecification.pdf) +- [Sentinel-1 Product Specification v3.7 27 February 2020 S1-RS-MDA-52-7441 documenting IPF 3.30](https://sentinels.copernicus.eu/documents/247904/1877131/Sentinel-1-Product-Specification) +- [Radiometric Calibration of S-1 Level-1 Products Generated by the S-1 IPF v1.0 21/05/2015 ESA-EOPG-CSCOP-TN-0002](https://sentinels.copernicus.eu/documents/247904/685163/S1-Radiometric-Calibration-V1.0.pdf) ## Design decisions diff --git a/docs/DATATREE.md b/docs/DATATREE.md index d31fdea..e14bd00 100644 --- a/docs/DATATREE.md +++ b/docs/DATATREE.md @@ -213,7 +213,8 @@ What we do in each case follows the decisions above. ## Plan One PR with tests per step. Step 1 comes first, steps 2 to 9 are independent and each -adds converters to the step 1 registry, steps 10 and 11 come last. +adds converters to the step 1 registry, steps 10 and 11 come last, step 12 can be +done at any time. 1. Done: skeleton and test harness. - `eopf_product.open_datatree` reads each XML once and opens each measurement once. @@ -295,3 +296,15 @@ adds converters to the step 1 registry, steps 10 and 11 come last. environment (the test SAFEs lack the `support/*.xsd` the CPM needs), compare structure exactly and values within tolerance, list the deviations due to the quirks. Optionally open our Zarr output with `xarray-eopf`. +1. Parse once: decode each XML once and pass the result down. + - Manifest: `build_stac_discovery` and `build_processing_history` take the parsed + `ElementTree` root instead of the path, now `build_stac_discovery` parses the + manifest again for every node. + - Annotation, calibration and noise: `open_pol_dataset` and the `sentinel1` + openers of calibration, noise, replicas and `azimuth_fm_rate` still decode the + whole XML at every `parse_tag` or `parse_tag_as_list` call, the wrappers of the + `eopf_sentinel1` bases call `parse_xml` once per call. Switch them to the + `parse_xml` dict and `get_tag`. + - Then `open_datatree` keeps the parsed dicts instead of `BytesIO`, the manifest + keeps its `BytesIO` for `parse_manifest_sentinel1` until the manifest XSD + decode in [Open points](#open-points) is settled. diff --git a/xarray_sentinel/conventions.py b/xarray_sentinel/conventions.py index ee77f3b..5611c91 100644 --- a/xarray_sentinel/conventions.py +++ b/xarray_sentinel/conventions.py @@ -1,7 +1,7 @@ """CF representation of metadata according to Sentinel-1 Product Specification. See: S1-RS-MDA-52-7441, DI-MPC-PB, MPC-0240, 3/7, 27/02/2020 -https://sentinel.esa.int/documents/247904/1877131/Sentinel-1-Product-Specification +https://sentinels.copernicus.eu/documents/247904/1877131/Sentinel-1-Product-Specification """ import xarray as xr diff --git a/xarray_sentinel/eopf_sentinel1.py b/xarray_sentinel/eopf_sentinel1.py index 22d726a..b2841ea 100644 --- a/xarray_sentinel/eopf_sentinel1.py +++ b/xarray_sentinel/eopf_sentinel1.py @@ -1,3 +1,17 @@ +"""Convert the Sentinel-1 SAFE XML files to the datasets of the EOPF layout. + +References +---------- + - Sentinel-1 documents: + https://sentiwiki.copernicus.eu/web/s1-documents + - Sentinel-1 Product Specification 3/16/3 14 November 2025 S1-RS-MDA-52-7441 documenting IPF 4.00 + https://sentiwiki.copernicus.eu/__attachments/a_8ed8a4b44eb1585570c8ce755c81f47149b5710b878272ab3b40e9cfa2c9e046/S1-RS-MDA-52-7441-Sentinel-1-Product-Specification-2025-3.16.3.pdf + - EOPF CPM 3.0, the SAFE to Zarr converter that defines the target layout, see docs/DATATREE.md + https://pypi.org/project/eopf/3.0.0/ + - Sentinel-1 L1 and L2 Zarr product format of the EOPF S1 L12 reference processor + https://s1.pages.eopf.copernicus.eu/s1-l12-rp/main/pfs/product_format_overview.html +""" + import warnings from typing import Any diff --git a/xarray_sentinel/sentinel1.py b/xarray_sentinel/sentinel1.py index d235269..cf92a0d 100644 --- a/xarray_sentinel/sentinel1.py +++ b/xarray_sentinel/sentinel1.py @@ -2,12 +2,12 @@ References ---------- - - Sentinel-1 document library: - https://sentinels.copernicus.eu/web/sentinel/user-guides/sentinel-1-sar/document-library + - Sentinel-1 documents: + https://sentiwiki.copernicus.eu/web/s1-documents - Sentinel-1 Product Specification v3.9 07 May 2021 S1-RS-MDA-52-7441-3-9 documenting IPF 3.40 - https://sentinel.esa.int/documents/247904/1877131/S1-RS-MDA-52-7441-3-9-2_Sentinel-1ProductSpecification.pdf + https://sentinels.copernicus.eu/documents/247904/1877131/S1-RS-MDA-52-7441-3-9-2_Sentinel-1ProductSpecification.pdf - Sentinel-1 Product Specification v3.7 27 February 2020 S1-RS-MDA-52-7441 documenting IPF 3.30 - https://sentinel.esa.int/documents/247904/1877131/Sentinel-1-Product-Specification + https://sentinels.copernicus.eu/documents/247904/1877131/Sentinel-1-Product-Specification """ from __future__ import annotations