diff --git a/openeo_driver/util/stac_utils.py b/openeo_driver/util/stac_utils.py index 4b078a6a..b17dd282 100644 --- a/openeo_driver/util/stac_utils.py +++ b/openeo_driver/util/stac_utils.py @@ -106,9 +106,11 @@ def recurse(item_path: Union[str, Path]) -> list: href = robust_urljoin(item_path, href) if "rel" in link and (link["rel"] == "child" or link["rel"] == "item"): - all_files.extend(get_files_from_stac_catalog(href, include_metadata)) - else: + all_files.extend(recurse(href)) + elif ("rel" not in link) or (link["rel"] != "collection"): all_files.append(href) + else: + pass return all_files return_files = recurse(catalog_path) diff --git a/openeo_driver/views.py b/openeo_driver/views.py index a4e21363..b371e9d1 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -1850,7 +1850,7 @@ def _asset_object( if filename.endswith(".model"): # Machine learning models. return result_dict - bands = asset_metadata.get("bands") + bands = asset_metadata.get("bands") or asset_metadata.get("eo:bands") if bands: # TODO: #298 this is a quick stop-gap solution for lack of clear API diff --git a/tests/data/example_stac_catalog_11/1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z.json b/tests/data/example_stac_catalog_11/1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z.json new file mode 100644 index 00000000..c7df20ed --- /dev/null +++ b/tests/data/example_stac_catalog_11/1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z.json @@ -0,0 +1,111 @@ +{ + "assets": { + "1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1774.0, + "mean": 380.50094473311, + "minimum": 171.0, + "stddev": 176.19790726983, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1470.0, + "mean": 592.27586206896, + "minimum": 282.0, + "stddev": 138.20284171799, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1190.0, + "mean": 412.22508266415, + "minimum": 270.0, + "stddev": 103.22080164099, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-06Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-06Z.tif", + "type": "image/tiff; application=geotiff" + } + }, + "bbox": [ + 5.069386757303357, + 51.214543255519786, + 5.080663210429039, + 51.22048989375812 + ], + "collection": "j-2606111953054049bc2368898439491e", + "epsg": 32631, + "geometry": { + "coordinates": [ + [ + [ + 5.0804038775948115, + 51.214543255519786 + ], + [ + 5.080663210429039, + 51.22029444279013 + ], + [ + 5.069644719194295, + 51.22048989375812 + ], + [ + 5.069386757303357, + 51.21473866656907 + ], + [ + 5.0804038775948115, + 51.214543255519786 + ] + ] + ], + "type": "Polygon" + }, + "id": "1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z", + "links": [ + { + "href": "./collection.json", + "rel": "collection", + "type": "application/json" + } + ], + "properties": { + "datetime": "2023-06-06T00:00:00Z", + "proj:code": "EPSG:32631", + "proj:epsg": 32631 + }, + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/projection/v1.2.0/schema.json" + ], + "stac_version": "1.1.0", + "type": "Feature" +} \ No newline at end of file diff --git a/tests/data/example_stac_catalog_11/75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z.json b/tests/data/example_stac_catalog_11/75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z.json new file mode 100644 index 00000000..1a42d34d --- /dev/null +++ b/tests/data/example_stac_catalog_11/75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z.json @@ -0,0 +1,111 @@ +{ + "assets": { + "75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1766.0, + "mean": 401.07085498347, + "minimum": 180.0, + "stddev": 162.43588375276, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1336.0, + "mean": 598.11572980633, + "minimum": 278.0, + "stddev": 133.74416960473, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1072.0, + "mean": 368.17264997638, + "minimum": 216.0, + "stddev": 99.2670146459, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-01Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-01Z.tif", + "type": "image/tiff; application=geotiff" + } + }, + "bbox": [ + 5.069386757303357, + 51.214543255519786, + 5.080663210429039, + 51.22048989375812 + ], + "collection": "j-2606111953054049bc2368898439491e", + "epsg": 32631, + "geometry": { + "coordinates": [ + [ + [ + 5.0804038775948115, + 51.214543255519786 + ], + [ + 5.080663210429039, + 51.22029444279013 + ], + [ + 5.069644719194295, + 51.22048989375812 + ], + [ + 5.069386757303357, + 51.21473866656907 + ], + [ + 5.0804038775948115, + 51.214543255519786 + ] + ] + ], + "type": "Polygon" + }, + "id": "75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z", + "links": [ + { + "href": "./collection.json", + "rel": "collection", + "type": "application/json" + } + ], + "properties": { + "datetime": "2023-06-01T00:00:00Z", + "proj:code": "EPSG:32631", + "proj:epsg": 32631 + }, + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/projection/v1.2.0/schema.json" + ], + "stac_version": "1.1.0", + "type": "Feature" +} \ No newline at end of file diff --git a/tests/data/example_stac_catalog_11/collection.json b/tests/data/example_stac_catalog_11/collection.json new file mode 100644 index 00000000..ad93b465 --- /dev/null +++ b/tests/data/example_stac_catalog_11/collection.json @@ -0,0 +1,240 @@ +{ + "assets": { + "1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1774.0, + "mean": 380.50094473311, + "minimum": 171.0, + "stddev": 176.19790726983, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1470.0, + "mean": 592.27586206896, + "minimum": 282.0, + "stddev": 138.20284171799, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1190.0, + "mean": 412.22508266415, + "minimum": 270.0, + "stddev": 103.22080164099, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-06Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-06Z.tif", + "type": "image/tiff; application=geotiff" + }, + "75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1766.0, + "mean": 401.07085498347, + "minimum": 180.0, + "stddev": 162.43588375276, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1336.0, + "mean": 598.11572980633, + "minimum": 278.0, + "stddev": 133.74416960473, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1072.0, + "mean": 368.17264997638, + "minimum": 216.0, + "stddev": 99.2670146459, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-01Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-01Z.tif", + "type": "image/tiff; application=geotiff" + }, + "d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1628.0, + "mean": 360.10746339159, + "minimum": 174.0, + "stddev": 160.47737013663, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1358.0, + "mean": 546.93197921587, + "minimum": 263.0, + "stddev": 129.15120454283, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1116.0, + "mean": 344.86820973075, + "minimum": 200.0, + "stddev": 95.975778884601, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-04Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-04Z.tif", + "type": "image/tiff; application=geotiff" + } + }, + "description": "test description", + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-06-01T00:00:00Z", + "2023-06-06T00:00:00Z" + ] + ] + } + }, + "id": "j-2606111953054049bc2368898439491e", + "license": "proprietary", + "links": [ + { + "href": "./d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z.json", + "rel": "item", + "type": "application/geo+json" + }, + { + "href": "./1b442790-2231-4dd1-b9f4-d3550a512cd1_2023-06-06T00:00:00Z.json", + "rel": "item", + "type": "application/geo+json" + }, + { + "href": "./75625346-7e28-433f-9918-8318369ff7ad_2023-06-01T00:00:00Z.json", + "rel": "item", + "type": "application/geo+json" + } + ], + "openeo:status": "finished", + "providers": [ + { + "description": "This data was processed on an openEO backend maintained by VITO.", + "name": "VITO", + "processing:expression": { + "expression": { + "loadstac1": { + "arguments": { + "url": "https://raw.githubusercontent.com/Open-EO/openeo-geopyspark-driver/refs/heads/master/docker/local_batch_job/example_stac_catalog/collection.json" + }, + "process_id": "load_stac", + "result": true + } + }, + "format": "openeo" + }, + "processing:facility": "openEO Geotrellis backend", + "processing:software": { + "Geotrellis backend": "0.72.0a13" + }, + "roles": [ + "processor" + ] + } + ], + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/projection/v1.2.0/schema.json" + ], + "stac_version": "1.1.0", + "summaries": {}, + "title": "server_test.py_stac-version_1.1", + + "properties": { + "datetime": "2023-06-01T00:00:00Z" + }, + "type": "Collection" +} \ No newline at end of file diff --git a/tests/data/example_stac_catalog_11/d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z.json b/tests/data/example_stac_catalog_11/d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z.json new file mode 100644 index 00000000..a31b525b --- /dev/null +++ b/tests/data/example_stac_catalog_11/d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z.json @@ -0,0 +1,111 @@ +{ + "assets": { + "d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z_openEO": { + "bands": [ + { + "name": "B04", + "statistics": { + "maximum": 1628.0, + "mean": 360.10746339159, + "minimum": 174.0, + "stddev": 160.47737013663, + "valid_percent": 85.92 + } + }, + { + "name": "B03", + "statistics": { + "maximum": 1358.0, + "mean": 546.93197921587, + "minimum": 263.0, + "stddev": 129.15120454283, + "valid_percent": 85.92 + } + }, + { + "name": "B02", + "statistics": { + "maximum": 1116.0, + "mean": 344.86820973075, + "minimum": 200.0, + "stddev": 95.975778884601, + "valid_percent": 85.92 + } + } + ], + "href": "./openEO_2023-06-04Z.tif", + "proj:bbox": [ + 644530.0, + 5675740.0, + 645300.0, + 5676380.0 + ], + "proj:code": "EPSG:32631", + "proj:epsg": 32631, + "proj:shape": [ + 64, + 77 + ], + "roles": [ + "data" + ], + "title": "s3://openeo-data-dev-waw4-1/batch_jobs/j-2606111953054049bc2368898439491e/openEO_2023-06-04Z.tif", + "type": "image/tiff; application=geotiff" + } + }, + "bbox": [ + 5.069386757303357, + 51.214543255519786, + 5.080663210429039, + 51.22048989375812 + ], + "collection": "j-2606111953054049bc2368898439491e", + "epsg": 32631, + "geometry": { + "coordinates": [ + [ + [ + 5.0804038775948115, + 51.214543255519786 + ], + [ + 5.080663210429039, + 51.22029444279013 + ], + [ + 5.069644719194295, + 51.22048989375812 + ], + [ + 5.069386757303357, + 51.21473866656907 + ], + [ + 5.0804038775948115, + 51.214543255519786 + ] + ] + ], + "type": "Polygon" + }, + "id": "d3e7a187-1199-461a-8cd4-3f2922cd19a2_2023-06-04T00:00:00Z", + "links": [ + { + "href": "./collection.json", + "rel": "collection", + "type": "application/json" + } + ], + "properties": { + "datetime": "2023-06-04T00:00:00Z", + "proj:code": "EPSG:32631", + "proj:epsg": 32631 + }, + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/projection/v1.2.0/schema.json" + ], + "stac_version": "1.1.0", + "type": "Feature" +} \ No newline at end of file diff --git a/tests/data/example_stac_catalog_11/openEO_2023-06-01Z.tif b/tests/data/example_stac_catalog_11/openEO_2023-06-01Z.tif new file mode 100644 index 00000000..bd33c9e0 Binary files /dev/null and b/tests/data/example_stac_catalog_11/openEO_2023-06-01Z.tif differ diff --git a/tests/data/example_stac_catalog_11/openEO_2023-06-04Z.tif b/tests/data/example_stac_catalog_11/openEO_2023-06-04Z.tif new file mode 100644 index 00000000..3b09d374 Binary files /dev/null and b/tests/data/example_stac_catalog_11/openEO_2023-06-04Z.tif differ diff --git a/tests/data/example_stac_catalog_11/openEO_2023-06-06Z.tif b/tests/data/example_stac_catalog_11/openEO_2023-06-06Z.tif new file mode 100644 index 00000000..16476766 Binary files /dev/null and b/tests/data/example_stac_catalog_11/openEO_2023-06-06Z.tif differ diff --git a/tests/data/job_metadata_from_cwl.json b/tests/data/job_metadata_from_cwl.json new file mode 100644 index 00000000..e1c01fc3 --- /dev/null +++ b/tests/data/job_metadata_from_cwl.json @@ -0,0 +1,308 @@ +{ + "geometry": null, + "bbox": null, + "area": null, + "start_datetime": "2023-06-01T00:00:00Z", + "end_datetime": "2023-06-06T00:00:00Z", + "links": [ + { + "href": ".../collection.json", + "rel": "original", + "title": "Link to original STAC catalog.", + "type": "application/json" + } + ], + "items": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "openEO_2023-06-01Z.tif", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [5.069685009564396, 51.21481793134414], + [5.069685009564396, 51.22021524164682], + [5.080364860947119, 51.22021524164682], + [5.080364860947119, 51.21481793134414], + [5.069685009564396, 51.21481793134414] + ] + ] + }, + "bbox": [ + 5.069685009564396, + 51.21481793134414, + 5.080364860947119, + 51.22021524164682 + ], + "properties": { + "testing_custom_property": "https://github.com/cloudinsar/s1-workflows/issues/70", + "datetime": "2023-06-01T00:00:00Z" + }, + "links": [], + "assets": { + "openEO_2023-06-01Z.tif": { + "href": ".../openEO_2023-06-01Z.tif", + "roles": ["data"], + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B04", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B03", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B02", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + } + ], + "raster:bands": [ + { + "name": "B04", + "statistics": { + "minimum": 180, + "maximum": 1766, + "mean": 401.07085498347, + "stddev": 162.43588375276, + "valid_percent": 100 + } + }, + { + "name": "B03", + "statistics": { + "minimum": 278, + "maximum": 1336, + "mean": 598.11572980633, + "stddev": 133.74416960473, + "valid_percent": 100 + } + }, + { + "name": "B02", + "statistics": { + "minimum": 216, + "maximum": 1072, + "mean": 368.17264997638, + "stddev": 99.2670146459, + "valid_percent": 100 + } + } + ] + } + } + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "openEO_2023-06-04Z.tif", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [5.069685009564396, 51.21481793134414], + [5.069685009564396, 51.22021524164682], + [5.080364860947119, 51.22021524164682], + [5.080364860947119, 51.21481793134414], + [5.069685009564396, 51.21481793134414] + ] + ] + }, + "bbox": [ + 5.069685009564396, + 51.21481793134414, + 5.080364860947119, + 51.22021524164682 + ], + "properties": {"datetime": "2023-06-04T00:00:00Z"}, + "links": [], + "assets": { + "openEO_2023-06-04Z.tif": { + "href": ".../openEO_2023-06-04Z.tif", + "roles": ["data"], + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B04", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B03", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B02", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + } + ], + "raster:bands": [ + { + "name": "B04", + "statistics": { + "minimum": 174, + "maximum": 1628, + "mean": 360.10746339159, + "stddev": 160.47737013663, + "valid_percent": 100 + } + }, + { + "name": "B03", + "statistics": { + "minimum": 263, + "maximum": 1358, + "mean": 546.93197921587, + "stddev": 129.15120454283, + "valid_percent": 100 + } + }, + { + "name": "B02", + "statistics": { + "minimum": 200, + "maximum": 1116, + "mean": 344.86820973075, + "stddev": 95.975778884601, + "valid_percent": 100 + } + } + ] + } + } + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "openEO_2023-06-06Z.tif", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [5.069685009564396, 51.21481793134414], + [5.069685009564396, 51.22021524164682], + [5.080364860947119, 51.22021524164682], + [5.080364860947119, 51.21481793134414], + [5.069685009564396, 51.21481793134414] + ] + ] + }, + "bbox": [ + 5.069685009564396, + 51.21481793134414, + 5.080364860947119, + 51.22021524164682 + ], + "properties": {"datetime": "2023-06-06T00:00:00Z"}, + "links": [], + "assets": { + "openEO_2023-06-06Z.tif": { + "href": ".../openEO_2023-06-06Z.tif", + "roles": ["data"], + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B04", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B03", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + }, + { + "name": "B02", + "common_name": null, + "wavelength_um": null, + "aliases": null, + "gsd": null + } + ], + "raster:bands": [ + { + "name": "B04", + "statistics": { + "minimum": 171, + "maximum": 1774, + "mean": 380.50094473311, + "stddev": 176.19790726983, + "valid_percent": 100 + } + }, + { + "name": "B03", + "statistics": { + "minimum": 282, + "maximum": 1470, + "mean": 592.27586206896, + "stddev": 138.20284171799, + "valid_percent": 100 + } + }, + { + "name": "B02", + "statistics": { + "minimum": 270, + "maximum": 1190, + "mean": 412.22508266415, + "stddev": 103.22080164099, + "valid_percent": 100 + } + } + ] + } + } + } + ], + "epsg": null, + "instruments": [], + "processing:facility": "VITO - SPARK", + "processing:software": "openeo-geotrellis-0.72.0a12", + "unique_process_ids": ["run_cwl_to_stac"], + "providers": [ + { + "name": "VITO", + "description": "This data was processed on an openEO backend maintained by VITO.", + "roles": ["processor"], + "processing:facility": "openEO Geotrellis backend", + "processing:software": {"Geotrellis backend": "0.72.0a12"}, + "processing:expression": { + "format": "openeo", + "expression": { + "runcwltostac1": { + "process_id": "run_cwl_to_stac", + "arguments": { + "context": {}, + "cwl": "https://raw.githubusercontent.com/Open-EO/openeo-geotrellis-kubernetes/master/openeo-geopyspark-k8s-custom-processes/src/openeo_geopyspark_k8s_custom_processes/cwl/dummy_stac.cwl" + }, + "result": true + } + } + } + } + ] +} \ No newline at end of file diff --git a/tests/test_views.py b/tests/test_views.py index ab8a40dd..54cfcdf3 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1359,6 +1359,74 @@ def test_collections_queryables(self, api120): class TestBatchJobs: AUTH_HEADER = TEST_USER_AUTH_HEADER + @staticmethod + def load_dummy_batch_job_metadata(job_metadata_path: Path, job_id: str, full=True): + """ + Mocks a lot of metadata with random values + """ + from openeo.util import rfc3339 + result_metadata = json.loads(Path(job_metadata_path).read_text()) + process_graph = result_metadata.get("providers", [])[0]["processing:expression"]["expression"] + job_info: dict = { + "job_id": job_id, + "title": "load_dummy_batch_job_metadata title", + "description": "load_dummy_batch_job_metadata description", + "process": {"process_graph": process_graph}, + "status": "finished", + "progress": 100, + "created": "2020-06-11T11:44:44Z", + "updated": "2020-06-11T11:55:55Z", + "plan": "some_plan", + "costs": 1.666, + "budget": 4.666, + "usage": { + "cpu": {"value": 666, "unit": "cpu-seconds"}, + "duration": {"value": 6, "unit": "seconds"}, + "memory": {"value": 666666, "unit": "mb-seconds"}, + }, + "results_metadata": result_metadata, + "job_options": {}, + } + + def map_safe(prop: str, f): + value = job_info.get(prop) + return f(value) if value else None + + def get_results_metadata(result_metadata_prop: str): + return job_info.get("results_metadata", {}).get(result_metadata_prop) + + def map_results_metadata_safe(result_metadata_prop: str, f): + value = get_results_metadata(result_metadata_prop) + return f(value) if value is not None else None + + return BatchJobMetadata( + id=job_info["job_id"], + status=job_info["status"], + created=map_safe("created", rfc3339.parse_datetime), + process=job_info.get("process") if full else None, + job_options=job_info.get("job_options") if full else None, + title=job_info.get("title"), + description=job_info.get("description"), + updated=map_safe("updated", rfc3339.parse_datetime), + started=map_safe("started", rfc3339.parse_datetime), + finished=map_safe("finished", rfc3339.parse_datetime), + memory_time_megabyte=map_safe( + "memory_time_megabyte_seconds", lambda seconds: timedelta(seconds=seconds) + ), + cpu_time=map_safe("cpu_time_seconds", lambda seconds: timedelta(seconds=seconds)), + geometry=get_results_metadata("geometry"), + bbox=get_results_metadata("bbox"), + start_datetime=map_results_metadata_safe("start_datetime", rfc3339.parse_datetime), + end_datetime=map_results_metadata_safe("end_datetime", rfc3339.parse_datetime), + instruments=get_results_metadata("instruments"), + epsg=get_results_metadata("epsg"), + links=get_results_metadata("links"), + usage=job_info.get("usage"), + costs=job_info.get("costs"), + proj_shape=get_results_metadata("proj:shape"), + proj_bbox=get_results_metadata("proj:bbox"), + ) + @staticmethod @contextmanager def _fresh_job_registry(next_job_id="job-1234", output_root: Optional[Path] = None, jobs: Optional[dict] = None): @@ -1375,7 +1443,12 @@ def _fresh_job_registry(next_job_id="job-1234", output_root: Optional[Path] = No mock.patch.object(dummy_backend.DummyBatchJobs, "_output_root", return_value=output_root) ) + _server_test_metadata_path = get_path("job_metadata_from_cwl.json") + dummy_backend.DummyBatchJobs._job_registry = { + (TEST_USER, "j-job_metadata_from_cwl"): TestBatchJobs.load_dummy_batch_job_metadata( + _server_test_metadata_path, job_id="j-job_metadata_from_cwl" + ), (TEST_USER, "07024ee9-7847-4b8a-b260-6c879a2b3cdc"): BatchJobMetadata( id="07024ee9-7847-4b8a-b260-6c879a2b3cdc", status="running", @@ -1435,7 +1508,19 @@ def _fresh_job_registry(next_job_id="job-1234", output_root: Optional[Path] = No end_datetime=None, ), } - dummy_backend.DummyBatchJobs._job_result_registry = {} + if next_job_id == "j-job_metadata_from_cwl": + _server_test_raw = json.loads(_server_test_metadata_path.read_text()) + dummy_backend.DummyBatchJobs._job_result_registry = { + # ("j-job_metadata_from_cwl", TEST_USER): bjmd + ("j-job_metadata_from_cwl", TEST_USER): BatchJobResultMetadata( + assets={}, + items={item["id"]: item for item in _server_test_raw.get("items", [])}, + links=_server_test_raw.get("links", []), + providers=_server_test_raw.get("providers", []), + ), + } + else: + dummy_backend.DummyBatchJobs._job_result_registry = {} if jobs: for job_id, job_settings in jobs.items(): @@ -1704,6 +1789,16 @@ def test_list_user_jobs_100(self, api100): resp = api100.get("/jobs", headers=self.AUTH_HEADER) assert resp.assert_status_code(200).json == { "jobs": [ + { + "costs": 1.666, + "created": "2020-06-11T11:44:44Z", + "description": "load_dummy_batch_job_metadata description", + "id": "j-job_metadata_from_cwl", + "progress": 100, + "status": "finished", + "title": "load_dummy_batch_job_metadata title", + "updated": "2020-06-11T11:55:55Z", + }, { "id": "07024ee9-7847-4b8a-b260-6c879a2b3cdc", "status": "running", @@ -2314,6 +2409,22 @@ def test_get_job_results_110(self, api110): } ) + def test_get_job_results_100_from_cwl(self, api100): + with self._fresh_job_registry(next_job_id="j-job_metadata_from_cwl"): + resp = api100.get("/jobs/j-job_metadata_from_cwl/results", headers=self.AUTH_HEADER) + resp_txt = resp.text + assert "j-job_metadata_from_cwl" in resp_txt + # assert "bands" in resp_txt # TODO? + pystac.Item.from_dict(json.loads(resp_txt)) + + def test_get_job_results_110_from_cwl(self, api110): + with self._fresh_job_registry(next_job_id="j-job_metadata_from_cwl"): + resp = api110.get("/jobs/j-job_metadata_from_cwl/results", headers=self.AUTH_HEADER) + resp_txt = resp.text + assert "j-job_metadata_from_cwl" in resp_txt + assert "bands" in resp_txt + pystac.Collection.from_dict(json.loads(resp_txt)) + def test_get_job_results_public_href_asset_100(self, api, backend_implementation): import numpy as np diff --git a/tests/util/test_stac_utils.py b/tests/util/test_stac_utils.py index bb6aed22..24da85b6 100644 --- a/tests/util/test_stac_utils.py +++ b/tests/util/test_stac_utils.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pystac import pytest from pystac import Collection @@ -19,6 +20,7 @@ _TEST_BUCKET = "test-stac-bucket" example_stac_catalog_dir = repository_root / "tests/data/example_stac_catalog" +example_stac_catalog_dir_11 = repository_root / "tests/data/example_stac_catalog_11" @pytest.fixture() @@ -54,6 +56,21 @@ def test_get_files_from_stac_catalog_path_include_metadata(): assert len(ret) == 7 +def test_get_files_from_stac_catalog_path_11(): + stac_root = example_stac_catalog_dir_11 / "collection.json" + ret = get_files_from_stac_catalog(stac_root) + ret = set(ret) + print(ret) + assert len(ret) == 3 + + +def test_get_files_from_stac_catalog_path_include_metadata_11(): + stac_root = example_stac_catalog_dir_11 / "collection.json" + ret = get_files_from_stac_catalog(stac_root, include_metadata=True) + ret = set(ret) + print(ret) + assert len(ret) == 7 + def test_get_files_from_stac_catalog_url(): stac_root = "https://raw.githubusercontent.com/Open-EO/openeo-geopyspark-driver/refs/heads/master/docker/local_batch_job/example_stac_catalog/collection.json" ret = get_files_from_stac_catalog(stac_root) @@ -72,6 +89,7 @@ def test_get_files_from_stac_catalog_url_include_metadata(): def test_get_assets_from_stac_catalog(): stac_root = example_stac_catalog_dir / "collection.json" + pystac.Collection.from_file(str(stac_root)).validate_all() ret = get_assets_from_stac_catalog(stac_root) print(ret) assert len(ret.values()) == 3 @@ -84,6 +102,20 @@ def test_get_items_from_stac_catalog(): assert len(ret) == 3 +def test_get_assets_from_stac_catalog_11(): + stac_root = example_stac_catalog_dir_11 / "collection.json" + pystac.Collection.from_file(str(stac_root)).validate_all() + ret = get_assets_from_stac_catalog(stac_root) + print(ret) + assert len(ret.values()) == 3 + + +def test_get_items_from_stac_catalog_11(): + stac_root = example_stac_catalog_dir_11 / "collection.json" + ret = get_items_from_stac_catalog(stac_root) + print(ret) + assert len(ret) >= 3 # STAC 1.1 has an extra item on root level + def test_get_items_from_stac_catalog_recursive(): stac_root = str(repository_root / "tests/data/recursive-stac-example/collection.json") ret = get_items_from_stac_catalog(stac_root, make_hrefs_absolute=True)