From 709dd387ddb95971f7d76e242cae4800cd29c624 Mon Sep 17 00:00:00 2001 From: Jeffrey Seifried Date: Thu, 18 Jun 2026 09:13:54 -0700 Subject: [PATCH] migrate from GEE to xarray pipeline --- .env.example | 5 + .github/workflows/ci.yml | 2 +- .gitignore | 3 + .pre-commit-config.yaml | 23 +- ATTRIBUTION.md | 2 +- LICENSE.txt | 10 +- README.md | 62 +- jdluc/__tests__/__init__.py | 1 - jdluc/__tests__/conftest.py | 8 - jdluc/__tests__/test_attribution.py | 140 ++ jdluc/__tests__/test_datasets.py | 37 + jdluc/__tests__/test_emissions.py | 248 +++ jdluc/__tests__/test_emissions_factors.py | 82 + jdluc/__tests__/test_extract_county_fips.py | 197 -- .../test_extract_county_fips_integration.py | 149 -- jdluc/__tests__/test_extract_extract.py | 212 -- jdluc/__tests__/test_extract_gfw_peatlands.py | 359 ---- jdluc/__tests__/test_extract_harris_agb.py | 292 --- .../test_extract_harris_agb_integration.py | 95 - jdluc/__tests__/test_extract_huang_bgb.py | 108 - jdluc/__tests__/test_extract_integration.py | 120 -- .../test_extract_ipcc_climate_zones.py | 201 -- jdluc/__tests__/test_extract_mirror.py | 181 -- jdluc/__tests__/test_extract_nass_yields.py | 255 --- jdluc/__tests__/test_gcs.py | 16 + jdluc/__tests__/test_geo.py | 64 + jdluc/__tests__/test_harmonize.py | 113 + jdluc/__tests__/test_pipeline.py | 245 --- jdluc/__tests__/test_publish_bigquery.py | 287 --- .../test_publish_constants_and_shapes.py | 95 - jdluc/__tests__/test_publish_integration.py | 188 -- jdluc/__tests__/test_tiling.py | 106 + jdluc/__tests__/test_transform.py | 263 --- .../test_transform_emissions_integration.py | 409 ---- .../test_transform_land_use_integration.py | 306 --- .../test_transform_multistate_integration.py | 345 --- .../test_transform_summary_tables.py | 343 --- ...st_transform_summary_tables_integration.py | 473 ----- jdluc/__tests__/test_utils.py | 35 + .../__tests__/test_utils_asset_management.py | 111 - .../__tests__/test_utils_constants_extract.py | 145 -- jdluc/__tests__/test_utils_gee.py | 166 -- jdluc/__tests__/test_utils_transitions.py | 35 - jdluc/__tests__/test_utils_version.py | 99 - jdluc/attribution.py | 289 +++ jdluc/cli.py | 151 -- jdluc/config.py | 32 + jdluc/datasets/__init__.py | 54 + jdluc/datasets/base.py | 212 ++ jdluc/datasets/gfw_global_peatlands.py | 47 + jdluc/datasets/gfw_harris_agb.py | 49 + jdluc/datasets/glad_glcluc.py | 107 + jdluc/datasets/huang_bgb.py | 85 + jdluc/datasets/ipcc_climate_zones.py | 55 + jdluc/datasets/soilgrids_ocs.py | 56 + jdluc/datasets/usda_nass_cdl.py | 251 +++ jdluc/datasets/usda_nass_quickstats.py | 155 ++ jdluc/datasets/worldbank_jurisdictions.py | 130 ++ jdluc/emissions.py | 448 ++++ jdluc/emissions_factors.py | 111 + jdluc/extract/__init__.py | 0 jdluc/extract/_tile_pipeline.py | 271 --- jdluc/extract/county_fips.py | 172 -- jdluc/extract/extract.py | 163 -- jdluc/extract/gfw_peatlands.py | 138 -- jdluc/extract/harris_agb.py | 102 - jdluc/extract/huang_bgb.py | 136 -- jdluc/extract/ipcc_climate_zones.py | 169 -- jdluc/extract/mirror.py | 136 -- jdluc/extract/nass_yields.py | 167 -- jdluc/gcs.py | 215 ++ jdluc/geo.py | 141 ++ jdluc/harmonize.py | 394 ++++ jdluc/ingest.py | 95 + jdluc/pipeline.py | 121 -- jdluc/publish/__init__.py | 0 jdluc/publish/bigquery.py | 260 --- jdluc/publish/bq_to_gcs.py | 83 - jdluc/publish/gcs.py | 182 -- jdluc/publish/publish.py | 275 --- jdluc/tiling.py | 455 ++++ jdluc/transform/__init__.py | 0 jdluc/transform/_export.py | 84 - jdluc/transform/emissions.py | 484 ----- jdluc/transform/land_use.py | 203 -- jdluc/transform/summary_tables.py | 1033 --------- jdluc/transform/transform.py | 311 --- jdluc/utils.py | 74 + jdluc/utils/__init__.py | 0 jdluc/utils/_ee_types.py | 14 - jdluc/utils/asset_management.py | 195 -- jdluc/utils/constants.py | 825 -------- jdluc/utils/gee.py | 393 ---- jdluc/utils/states.py | 23 - jdluc/utils/transitions.py | 54 - jdluc/utils/version.py | 72 - pyproject.toml | 45 +- specs/methodology.md | 117 +- specs/peatland_methodology_supplement.md | 32 +- specs/pipeline_tech_design.md | 603 +----- uv.lock | 1882 ++++++++--------- 101 files changed, 5471 insertions(+), 13516 deletions(-) create mode 100644 .env.example delete mode 100644 jdluc/__tests__/__init__.py delete mode 100644 jdluc/__tests__/conftest.py create mode 100644 jdluc/__tests__/test_attribution.py create mode 100644 jdluc/__tests__/test_datasets.py create mode 100644 jdluc/__tests__/test_emissions.py create mode 100644 jdluc/__tests__/test_emissions_factors.py delete mode 100644 jdluc/__tests__/test_extract_county_fips.py delete mode 100644 jdluc/__tests__/test_extract_county_fips_integration.py delete mode 100644 jdluc/__tests__/test_extract_extract.py delete mode 100644 jdluc/__tests__/test_extract_gfw_peatlands.py delete mode 100644 jdluc/__tests__/test_extract_harris_agb.py delete mode 100644 jdluc/__tests__/test_extract_harris_agb_integration.py delete mode 100644 jdluc/__tests__/test_extract_huang_bgb.py delete mode 100644 jdluc/__tests__/test_extract_integration.py delete mode 100644 jdluc/__tests__/test_extract_ipcc_climate_zones.py delete mode 100644 jdluc/__tests__/test_extract_mirror.py delete mode 100644 jdluc/__tests__/test_extract_nass_yields.py create mode 100644 jdluc/__tests__/test_gcs.py create mode 100644 jdluc/__tests__/test_geo.py create mode 100644 jdluc/__tests__/test_harmonize.py delete mode 100644 jdluc/__tests__/test_pipeline.py delete mode 100644 jdluc/__tests__/test_publish_bigquery.py delete mode 100644 jdluc/__tests__/test_publish_constants_and_shapes.py delete mode 100644 jdluc/__tests__/test_publish_integration.py create mode 100644 jdluc/__tests__/test_tiling.py delete mode 100644 jdluc/__tests__/test_transform.py delete mode 100644 jdluc/__tests__/test_transform_emissions_integration.py delete mode 100644 jdluc/__tests__/test_transform_land_use_integration.py delete mode 100644 jdluc/__tests__/test_transform_multistate_integration.py delete mode 100644 jdluc/__tests__/test_transform_summary_tables.py delete mode 100644 jdluc/__tests__/test_transform_summary_tables_integration.py create mode 100644 jdluc/__tests__/test_utils.py delete mode 100644 jdluc/__tests__/test_utils_asset_management.py delete mode 100644 jdluc/__tests__/test_utils_constants_extract.py delete mode 100644 jdluc/__tests__/test_utils_gee.py delete mode 100644 jdluc/__tests__/test_utils_transitions.py delete mode 100644 jdluc/__tests__/test_utils_version.py create mode 100644 jdluc/attribution.py delete mode 100644 jdluc/cli.py create mode 100644 jdluc/config.py create mode 100644 jdluc/datasets/__init__.py create mode 100644 jdluc/datasets/base.py create mode 100644 jdluc/datasets/gfw_global_peatlands.py create mode 100644 jdluc/datasets/gfw_harris_agb.py create mode 100644 jdluc/datasets/glad_glcluc.py create mode 100644 jdluc/datasets/huang_bgb.py create mode 100644 jdluc/datasets/ipcc_climate_zones.py create mode 100644 jdluc/datasets/soilgrids_ocs.py create mode 100644 jdluc/datasets/usda_nass_cdl.py create mode 100644 jdluc/datasets/usda_nass_quickstats.py create mode 100644 jdluc/datasets/worldbank_jurisdictions.py create mode 100644 jdluc/emissions.py create mode 100644 jdluc/emissions_factors.py delete mode 100644 jdluc/extract/__init__.py delete mode 100644 jdluc/extract/_tile_pipeline.py delete mode 100644 jdluc/extract/county_fips.py delete mode 100644 jdluc/extract/extract.py delete mode 100644 jdluc/extract/gfw_peatlands.py delete mode 100644 jdluc/extract/harris_agb.py delete mode 100644 jdluc/extract/huang_bgb.py delete mode 100644 jdluc/extract/ipcc_climate_zones.py delete mode 100644 jdluc/extract/mirror.py delete mode 100644 jdluc/extract/nass_yields.py create mode 100644 jdluc/gcs.py create mode 100644 jdluc/geo.py create mode 100644 jdluc/harmonize.py create mode 100644 jdluc/ingest.py delete mode 100644 jdluc/pipeline.py delete mode 100644 jdluc/publish/__init__.py delete mode 100644 jdluc/publish/bigquery.py delete mode 100644 jdluc/publish/bq_to_gcs.py delete mode 100644 jdluc/publish/gcs.py delete mode 100644 jdluc/publish/publish.py create mode 100644 jdluc/tiling.py delete mode 100644 jdluc/transform/__init__.py delete mode 100644 jdluc/transform/_export.py delete mode 100644 jdluc/transform/emissions.py delete mode 100644 jdluc/transform/land_use.py delete mode 100644 jdluc/transform/summary_tables.py delete mode 100644 jdluc/transform/transform.py create mode 100644 jdluc/utils.py delete mode 100644 jdluc/utils/__init__.py delete mode 100644 jdluc/utils/_ee_types.py delete mode 100644 jdluc/utils/asset_management.py delete mode 100644 jdluc/utils/constants.py delete mode 100644 jdluc/utils/gee.py delete mode 100644 jdluc/utils/states.py delete mode 100644 jdluc/utils/transitions.py delete mode 100644 jdluc/utils/version.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..336d64d --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +INGEST_BUCKET_NAME = 'cornerstone-ingest' +GCP_PROJECT = 'cornerstone-data' +NUMBER_OF_DASK_WORKERS = 12 +SCRATCH_BUCKET_NAME = 'cornerstone-scratch' +USDA_NASS_API_KEY = 'Replace me with value from https://quickstats.nass.usda.gov/api' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bba149..59449b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,4 +29,4 @@ jobs: - name: Install dependencies run: uv sync - name: pytest (unit only) - run: uv run pytest jdluc/__tests__/ -m "not integration" + run: uv run pytest jdluc/__tests__/ diff --git a/.gitignore b/.gitignore index 0fd80ac..689403a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ __pycache__/ # Notebook scratch — version-controlled notebooks live under specs/analyses/ only. *.ipynb .ipynb_checkpoints/ + +# Env +.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b20655..7c98a5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,17 @@ exclude: | (?x)^(specs/analyses/) repos: - # mypyc-compiled black is ~2x faster than the upstream wheel. - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.1 - hooks: - - id: black - language_version: python3.14 - args: [--skip-string-normalization] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.13 hooks: - - id: ruff - args: [--fix] - + - id: ruff-check + args: [--fix, "--select=F,I,UP"] + - id: ruff-format - repo: https://github.com/seddonym/import-linter rev: v2.11 hooks: - id: import-linter language: python - # Run mypy within the venv so that libraries are visible - repo: local hooks: @@ -29,3 +20,9 @@ repos: language: system entry: uv run mypy jdluc pass_filenames: false + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index d17bba9..833a1ed 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -2,4 +2,4 @@ Attribute all emissions factors or other data built on this pipeline as: "Corner You must prominently display this attribution in public-facing models, tools, datasets or any other applications that are generated from this code. This includes reimplementations of the code that are built primarily by passing the methodology and/or technical specs to a coding agent. -If the code has been modified in a way that changes the published data products, that should be clearly stated. \ No newline at end of file +If the code has been modified in a way that changes the published data products, that should be clearly stated. diff --git a/LICENSE.txt b/LICENSE.txt index f3442f6..9fd02c9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,16 +1,16 @@ -Cornerstone Data Pipeline Attribution License 1.0 +Cornerstone Data Pipeline Attribution License 1.0 Acceptance -In order to receive this license, you must agree to its rules. The rules of this license are both obligations under that agreement and conditions to your license. +In order to receive this license, you must agree to its rules. The rules of this license are both obligations under that agreement and conditions to your license. License -The licensor licenses you to do everything with this software that would otherwise infringe the licensor’s copyright in it, or any patent claims the licensor can license or becomes able to license. If you make any written claim that the software infringes or contributes to infringement of any patent, your license ends immediately. +The licensor licenses you to do everything with this software that would otherwise infringe the licensor’s copyright in it, or any patent claims the licensor can license or becomes able to license. If you make any written claim that the software infringes or contributes to infringement of any patent, your license ends immediately. -Notices on Software +Notices on Software If you provide a copy of the software to anyone else, or provide or display Produced Data to anyone else, you must provide a copy of this license or a link to https://github.com/cornerstone-data/jdluc/blob/main/LICENSE.txt. Notices on Produced Data -If you calculate data (including summaries or reports of data) using this software, you must deliver an attribution, in a form to be specified by the licensor, in a reasonable and conspicuous manner to anyone to whom you deliver or display the data. +If you calculate data (including summaries or reports of data) using this software, you must deliver an attribution, in a form to be specified by the licensor, in a reasonable and conspicuous manner to anyone to whom you deliver or display the data. Modifications If you modify the Software, you may apply other license terms to your modifications, but you can only provide the Software to others under the terms at least as protective of the rights of the licensor as the terms of this license. diff --git a/README.md b/README.md index 3683ea8..d3b7c03 100644 --- a/README.md +++ b/README.md @@ -16,17 +16,20 @@ The methodology and technical decisions in this repo are intended as a starting ## Data access -Emissions factors and summary statistics: +Harmonized datasets and emissions layers: -```shell -curl -O https://storage.googleapis.com/cornerstone-luc/cornerstone-data/jdluc/conus/tables/crops.csv -curl -O https://storage.googleapis.com/cornerstone-luc/cornerstone-data/jdluc/conus/tables/transitions.csv +```python +>>> import xarray +>>> harmonized = xarray.open_zarr("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/harmonize.zarr", consolidated=False) +>>> emissions = xarray.open_zarr("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/emissions.zarr", consolidated=False) ``` -Simple raster visualizations: +Emissions factors: -- [Land use transitions, crop, and peat masks](https://cornerstone-data.projects.earthengine.app/view/jdluc-landuse-conus-20260513) -- [Pixel-level emissions estimates](https://cornerstone-data.projects.earthengine.app/view/jdluc-emissions-conus-20260513) +```python +>>> import pandas +>>> emission_factors = pandas.read_parquet("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/emissions-factors.parquet") +``` The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Please follow the latest attribution guidance in ATTRIBUTION.md. @@ -41,55 +44,44 @@ Once you're ready to look under the hood: ### Getting set up -You'll need access to a GCP project with the Earth Engine and BigQuery APIs enabled. You'll also need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager. +You'll need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager, plus a GCP project with GCS access and a USDA NASS QuickStats API key. ```bash -# Set GCP_PROJECT to your GCP project ID (the same value you set in -# jdluc/utils/constants.py). -export GCP_PROJECT=cornerstone-data +# Copy the example env file and fill in your values. +cp .env.example .env +# Set INGEST_BUCKET_NAME, GCP_PROJECT, SCRATCH_BUCKET_NAME, and USDA_NASS_API_KEY in .env. # Sync Python dependencies into the project venv. uv sync -# Authenticate gcloud application-default credentials. +# Authenticate gcloud application-default credentials (for GCS access). gcloud auth application-default login --project "${GCP_PROJECT}" - -# Authenticate Earth Engine and point it at the same project. -uv run earthengine authenticate -uv run earthengine set_project "${GCP_PROJECT}" ``` -### Running the pipeline - -First, set GCP project info by editing the deployment configuration block at the top of `jdluc/utils/constants.py`. +Obtain a NASS QuickStats API key at https://quickstats.nass.usda.gov/api and set it as `USDA_NASS_API_KEY` in `.env`. -Second, create the project folder in Google Earth Engine: - -```shell -uv run earthengine --project=${GCP_PROJECT} create folder projects/${GCP_PROJECT}/assets/cornerstone-luc -``` +### Running the pipeline -Finally: +The pipeline runs as a sequence of per-stage entry points, parameterized by a tile set (`DELAWARE`, `CONUS`, `BAY_AREA`, `GFW`, or `WHOLE_WORLD`): ```bash -# Default: Delaware (single state, smallest test region) -uv run python jdluc/cli.py -v +# 1. Ingest each source dataset (positional args; --concurrency / --overwrite optional). +uv run python -m jdluc.ingest -# Multi-state (Iowa + Nebraska + South Dakota) -uv run python jdluc/cli.py --region great_plains_test -v +# 2. Harmonize ingested tiles onto the common grid. +uv run python jdluc/harmonize.py -# Full CONUS (longer — full 48 states + DC) -uv run python jdluc/cli.py --region conus -v +# 3. Compute per-pixel land-conversion emissions (writes zarr). +uv run python jdluc/emissions.py -# Force re-export even when an asset already exists at the target version -uv run python jdluc/cli.py --force -v +# 4. Build the emissions-factor table (Delaware example; --skip-glad-crop-filter optional). +uv run python jdluc/emissions_factors.py --admin-id USA008 ``` ### Tests ```bash -uv run pytest jdluc -m "not integration" # unit tests; offline -uv run pytest jdluc -m integration # GEE / BQ integration +uv run pytest jdluc ``` ### Linting diff --git a/jdluc/__tests__/__init__.py b/jdluc/__tests__/__init__.py deleted file mode 100644 index a6d6068..0000000 --- a/jdluc/__tests__/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Tests for high-resolution land use change emissions pipeline diff --git a/jdluc/__tests__/conftest.py b/jdluc/__tests__/conftest.py deleted file mode 100644 index 5af1e20..0000000 --- a/jdluc/__tests__/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Test configuration for luc_high_res tests. - -Integration tests under this package define their own session-scoped -``gee_init`` and ``delaware_geometry`` fixtures locally; only shared -constants (region anchors) live here. -""" - -DELAWARE_FIPS = '10' diff --git a/jdluc/__tests__/test_attribution.py b/jdluc/__tests__/test_attribution.py new file mode 100644 index 0000000..1865f26 --- /dev/null +++ b/jdluc/__tests__/test_attribution.py @@ -0,0 +1,140 @@ +import collections.abc + +import numpy +import pytest +import xarray + +from jdluc.attribution import Crop, JurisdictionalCropEmission +from jdluc.datasets.worldbank_jurisdictions import AdminLevel + + +def get_darray_for_data( + data: collections.abc.Sequence[collections.abc.Sequence[float]], +) -> xarray.DataArray: + arr = numpy.array(data) + y, x = arr.shape + return xarray.DataArray( + coords={"y": range(y), "x": range(x)}, data=arr, dims=("y", "x") + ) + + +def test_from_xarray() -> None: + dset = xarray.Dataset( + { + # 99s sit on non-crop pixels — they must be excluded by the mask. + "emissions-per-hectare:tco2e-per-ha": get_darray_for_data( + [[1, 2, 99], [99, 4, 99]] + ), + "hectares-per-pixel:ha": get_darray_for_data([[10, 10, 10], [10, 10, 10]]), + "gfw:global-peatlands:is-peatland": get_darray_for_data( + [[1, 0, 0], [0, 1, 0]] + ), + "land-class:2020": get_darray_for_data([[2] * 2] * 2), + "peatland-occupation:tco2e-per-ha": get_darray_for_data( + [[5, 5, 5], [5, 5, 5]] + ), + # crop pixels: (0,0), (0,1), (1,1); of those, peatland at (0,0) and (1,1). + "usda-nass:cdl:crop-class": get_darray_for_data([[1, 1, 0], [0, 1, 0]]), + } + ) + result = JurisdictionalCropEmission.from_dset( + admin_id="USA008", + admin_level=AdminLevel.PROVINCIAL, + crop=Crop.CORN, + dset=dset, + jurisdiction_name="Delaware", + skip_glad_crop_filter=False, + ) + assert result == JurisdictionalCropEmission( + admin_id="USA008", + admin_level="PROVINCIAL", + crop_hectares=30.0, # 3 crop pixels × 10 ha + crop_name="CORN", + jurisdiction_name="Delaware", + peatland_crop_hectares=20.0, # only (0,0) and (1,1) are crop AND peatland + peatland_occupation_emissions=150.0, # 3 crop pixels × (5 × 10) + total_emissions=70.0, # (1+2+4) × 10; non-crop 99s excluded + ) + + +def test_from_xarray_masked_out_by_glad() -> None: + dset = xarray.Dataset( + { + "emissions-per-hectare:tco2e-per-ha": get_darray_for_data([[1, 2], [3, 4]]), + "hectares-per-pixel:ha": get_darray_for_data([[10, 10], [10, 10]]), + "gfw:global-peatlands:is-peatland": get_darray_for_data([[1, 1], [1, 1]]), + "land-class:2020": get_darray_for_data([[0] * 2] * 2), + "peatland-occupation:tco2e-per-ha": get_darray_for_data([[5, 5], [5, 5]]), + "usda-nass:cdl:crop-class": get_darray_for_data([[1, 1], [1, 1]]), + } + ) + result = JurisdictionalCropEmission.from_dset( + admin_id="USA008", + admin_level=AdminLevel.PROVINCIAL, + crop=Crop.CORN, + dset=dset, + jurisdiction_name="Delaware", + skip_glad_crop_filter=False, + ) + assert result.crop_hectares == 0.0 + assert result.peatland_crop_hectares == 0.0 + assert result.peatland_occupation_emissions == 0.0 + assert result.total_emissions == 0.0 + + +@pytest.mark.parametrize("glad_value", (0, 1, 2)) +@pytest.mark.parametrize("skip_glad_crop_filter", (False, True)) +def test_from_xarray_with_no_crop_pixels( + glad_value: int, skip_glad_crop_filter: bool +) -> None: + dset = xarray.Dataset( + { + "emissions-per-hectare:tco2e-per-ha": get_darray_for_data([[1, 2], [3, 4]]), + "hectares-per-pixel:ha": get_darray_for_data([[10, 10], [10, 10]]), + "gfw:global-peatlands:is-peatland": get_darray_for_data([[1, 1], [1, 1]]), + "land-class:2020": get_darray_for_data([[glad_value] * 2] * 2), + "peatland-occupation:tco2e-per-ha": get_darray_for_data([[5, 5], [5, 5]]), + "usda-nass:cdl:crop-class": get_darray_for_data([[0, 0], [0, 0]]), + } + ) + result = JurisdictionalCropEmission.from_dset( + admin_id="USA008", + admin_level=AdminLevel.PROVINCIAL, + crop=Crop.CORN, + dset=dset, + jurisdiction_name="Delaware", + skip_glad_crop_filter=skip_glad_crop_filter, + ) + assert result.crop_hectares == 0.0 + assert result.peatland_crop_hectares == 0.0 + assert result.peatland_occupation_emissions == 0.0 + assert result.total_emissions == 0.0 + + +def test_from_constituents() -> None: + constituents = [ + JurisdictionalCropEmission( + admin_id="", + admin_level="", + crop_hectares=value, + crop_name="CROP", + jurisdiction_name="", + peatland_crop_hectares=value, + peatland_occupation_emissions=value, + total_emissions=value, + ) + for value in (1, 2, 3) + ] + result = JurisdictionalCropEmission.from_constituents( + admin_id="ADMIN_ID", + admin_level=AdminLevel.NATIONAL, + constituents=constituents, + jurisdiction_name="JURISDICTION_NAME", + ) + assert result.admin_id == "ADMIN_ID" + assert result.admin_level == AdminLevel.NATIONAL.name + assert result.crop_hectares == 6 + assert result.jurisdiction_name == "JURISDICTION_NAME" + assert result.peatland_crop_hectares == 6 + assert result.peatland_occupation_emissions == 6 + assert result.total_emissions == 6 diff --git a/jdluc/__tests__/test_datasets.py b/jdluc/__tests__/test_datasets.py new file mode 100644 index 0000000..212c00b --- /dev/null +++ b/jdluc/__tests__/test_datasets.py @@ -0,0 +1,37 @@ +import pytest + +from jdluc.datasets.glad_glcluc import flatten_ranges +from jdluc.datasets.worldbank_jurisdictions import get_ten_degree_tile_ids_for_admin_id + + +def test_flatten_ranges() -> None: + assert flatten_ranges(range(0, 5), range(5, 10), range(10, 15)) == list(range(15)) + + +@pytest.mark.integration +def test_get_ten_degree_tile_ids_for_country() -> None: + iso_a3 = "BRA" # Brazil + expected = ( + "00N_040W", + "00N_050W", + "00N_060W", + "00N_070W", + "00N_080W", + "10N_050W", + "10N_060W", + "10N_070W", + "10N_080W", + "10S_040W", + "10S_050W", + "10S_060W", + "10S_070W", + "10S_080W", + "20S_030W", + "20S_050W", + "20S_060W", + "30S_060W", + ) + assert ( + tuple(get_ten_degree_tile_ids_for_admin_id(admin_id=iso_a3, admin_level=0)) + == expected + ) diff --git a/jdluc/__tests__/test_emissions.py b/jdluc/__tests__/test_emissions.py new file mode 100644 index 0000000..b0bfda2 --- /dev/null +++ b/jdluc/__tests__/test_emissions.py @@ -0,0 +1,248 @@ +import collections +import collections.abc + +import numpy +import pytest +import xarray + +from jdluc.datasets.glad_glcluc import LandClass +from jdluc.datasets.ipcc_climate_zones import Zone +from jdluc.emissions import ( + CARBON_PER_BIOMASS_LIVE_WOOD, + CO2E_PER_CARBON, + get_belowground_carbon, + get_dead_organic_matter_carbon, + get_grassland_carbon, + get_hectares_per_pixel, + get_land_class, + get_linear_discounted_emissions, + get_mineral_soil_emissions, + get_peatland_occupation_emissions, + get_soil_emissions, + get_vegetation_carbon, + get_vegetation_emissions, +) + + +def get_darray_for_data( + data: collections.abc.Sequence[collections.abc.Sequence[int | float]], +) -> xarray.DataArray: + arr = numpy.array(data) + y, x = arr.shape + return xarray.DataArray( + coords={ + "y": range(y), + "x": range(x), + }, + data=arr, + dims=("y", "x"), + ) + + +@pytest.mark.parametrize( + ("data", "expected"), + ( + ([[0, 100]] * 2, {LandClass.GRASSLAND.value: 4}), + ([[25, 125]] * 2, {LandClass.FOREST.value: 4}), + ([[200, 207]] * 2, {LandClass.WATER.value: 4}), + ( + [[241, 244, 250, 254]], + { + LandClass.BUILT_UP.value: 1, + LandClass.CROPLAND.value: 1, + LandClass.SNOW_ICE.value: 1, + LandClass.OCEAN.value: 1, + }, + ), + ([[-1, 255, 1 << 10, numpy.nan]], {0: 4}), + ), +) +def test_get_land_class(data: list[list[int]], expected: dict[int, int]) -> None: + result = get_land_class(glad_class=get_darray_for_data(data=data)) + assert result.name is None + assert collections.Counter(result.data.ravel()) == expected + + +def test_get_belowground_carbon() -> None: + agb = [[0, 1, 2]] + bgb = [[2, 1, 0]] + result = get_belowground_carbon( + aboveground_biomass=get_darray_for_data(agb), + belowground_biomass=get_darray_for_data(bgb), + ) + assert result.name == "tcarbon-per-ha" + assert numpy.array_equal( + result.data.ravel(), numpy.array([2, 1, 0.5]) * CARBON_PER_BIOMASS_LIVE_WOOD + ) + + +def test_get_dead_organic_matter_carbon() -> None: + agb = [[0, 1]] * 4 + zones = [ + [Zone.TROPICAL_WET.value] * 2, + [Zone.BOREAL_DRY.value] * 2, + [-1] * 2, + [numpy.nan] * 2, + ] + result = get_dead_organic_matter_carbon( + aboveground_biomass=get_darray_for_data(data=agb), + climate_zones=get_darray_for_data(data=zones), # type: ignore + ) + assert result.name == "tcarbon-per-ha" + assert numpy.array_equal( + result, + numpy.array( + [ + [ + 0, + 0.5 * 0.06 + 0.37 * 0.01, + ], + [ + 0, + 0.5 * 0.08 + 0.37 * 0.04, + ], + [0, 0], + [0, 0], + ] + ), + ) + + +def test_get_grassland_carbon() -> None: + data = [[Zone.TROPICAL_WET.value, Zone.BOREAL_DRY.value, -1, numpy.nan]] + result = get_grassland_carbon(climate_zones=get_darray_for_data(data=data)) + assert result.name == "tcarbon-per-ha" + assert numpy.array_equal(result.data, [[18, 3, 0, 0]]) + + +def test_get_vegetation_carbon() -> None: + agb = [[0, 1], [2, 3], [4, 5]] + bgb = [[1, 1], [0, 0], [0, 0]] + dom = [[1, 1]] * 3 + grassland = [[1, 1]] * 3 + classes = [ + [LandClass.FOREST.value, LandClass.GRASSLAND.value], + [LandClass.FOREST.value, LandClass.GRASSLAND.value], + [LandClass.CROPLAND.value, LandClass.BUILT_UP.value], + ] + result = get_vegetation_carbon( + aboveground_carbon=get_darray_for_data(data=agb), + belowground_carbon=get_darray_for_data(data=bgb), + dead_organic_carbon=get_darray_for_data(data=dom), + grassland_carbon=get_darray_for_data(data=grassland), + land_class=get_darray_for_data(data=classes), + ) + assert result.name == "tcarbon-per-ha" + assert numpy.array_equal(result.data, numpy.array([[2, 1], [3, 1], [0, 0]])) + + +def test_get_vegetation_emissions() -> None: + after = [[0, 1, 2, numpy.nan]] + before = [[numpy.nan, 2, 1, 0]] + result = get_vegetation_emissions( + after=get_darray_for_data(data=after), before=get_darray_for_data(data=before) + ) + assert result.name == "tco2e-per-ha" + assert numpy.array_equal( + result.data, [[numpy.nan, CO2E_PER_CARBON, 0, numpy.nan]], equal_nan=True + ) + + +def test_get_mineral_soil_emissions() -> None: + soc = [[0] * 4, [1] * 4, [2] * 4] + zones = [[Zone.BOREAL_DRY.value, Zone.TROPICAL_WET.value, -1, numpy.nan]] * 3 + result = get_mineral_soil_emissions( + climate_zones=get_darray_for_data(data=zones), + soil_organic_carbon=get_darray_for_data(data=soc), + ) + assert result.name == "tco2e-per-ha" + assert numpy.array_equal( + result.data * 3, + numpy.array([[0, 0, 0, 0], [2.53, 1.87, 0, 0], [5.06, 3.74, 0, 0]]), + ) + + +def test_get_soil_emissions() -> None: + after = [ + [ + LandClass.FOREST.value, + LandClass.GRASSLAND.value, + LandClass.CROPLAND.value, + LandClass.CROPLAND.value, + ] + ] * 4 + before = [ + [LandClass.FOREST.value] * 4, + [LandClass.GRASSLAND.value] * 4, + [LandClass.CROPLAND.value] * 4, + [LandClass.CROPLAND.value] * 4, + ] + zones = [[Zone.TROPICAL_WET.value] * 4] * 4 + is_peatland = [[0, 0, 0, 1]] * 4 + soc = [[1] * 4] * 4 + result = get_soil_emissions( + after=get_darray_for_data(data=after), + before=get_darray_for_data(data=before), + climate_zones=get_darray_for_data(data=zones), + is_peatland=get_darray_for_data(data=is_peatland), + soil_organic_carbon=get_darray_for_data(data=soc), + ) + assert result.name == "tco2e-per-ha" + assert numpy.array_equal( + result.data, + numpy.array( + [ + [0, 0, 1.87 / 3, 621], + [0, 0, 1.87 / 3, 621], + [0, 0, 0, 0], + [0, 0, 0, 0], + ] + ), + ) + + +def test_get_peatland_occupation_emissions() -> None: + is_peatland = [[1] * 7] + land_class = [ + [ + LandClass.BUILT_UP.value, + LandClass.CROPLAND.value, + LandClass.FOREST.value, + LandClass.GRASSLAND.value, + LandClass.OCEAN.value, + LandClass.SNOW_ICE.value, + LandClass.WATER.value, + ] + ] + result = get_peatland_occupation_emissions( + is_peatland=get_darray_for_data(data=is_peatland), + year_to_land_class={ + 2020: get_darray_for_data(data=land_class), + }, + ) + assert result.name == "tco2e-per-ha" + assert numpy.array_equal(result.data, [[37.3, 37.3, 0, 0, 0, 0, 0]]) + + +def test_get_linear_discounted_emissions() -> None: + result = get_linear_discounted_emissions( + epoch_to_emissions={ + (2000, 2005): get_darray_for_data(data=[[1]]), + (2005, 2010): get_darray_for_data(data=[[2]]), + (2010, 2015): get_darray_for_data(data=[[3]]), + (2015, 2020): get_darray_for_data(data=[[4]]), + } + ) + assert result.name == "tco2e-per-ha" + assert numpy.array_equal(result.data, [[0.625]]) + + +def test_get_hectares_per_pixel() -> None: + result = get_hectares_per_pixel(darray=get_darray_for_data(data=[[0] * 2] * 2)) + assert result.name == "ha" + numpy.testing.assert_allclose( + result.data, + numpy.array( + [[1237126.38106379, 1237126.38106379], [1236937.9607238, 1236937.9607238]] + ), + ) diff --git a/jdluc/__tests__/test_emissions_factors.py b/jdluc/__tests__/test_emissions_factors.py new file mode 100644 index 0000000..3df1aa4 --- /dev/null +++ b/jdluc/__tests__/test_emissions_factors.py @@ -0,0 +1,82 @@ +import numpy +import pandas + +from jdluc.emissions_factors import merge_emissions_and_yields + +EMISSIONS = pandas.DataFrame.from_records( + [ + # normal row + ("PROVINCIAL", "CORN", "Delaware", "USA008", 100.0, 10.0, 50.0, 200.0), + # zero production (crop_hectares == 0) + ("PROVINCIAL", "SOYBEANS", "Delaware", "USA008", 0.0, 0.0, 0.0, 80.0), + # unmatched yield + zero total_emissions + ("PROVINCIAL", "WHEAT", "Iowa", "USA016", 50.0, 0.0, 0.0, 0.0), + ], + columns=[ + "admin_level", + "crop_name", + "jurisdiction_name", + "admin_id", + "crop_hectares", + "peatland_crop_hectares", + "peatland_occupation_emissions", + "total_emissions", + ], +).set_index(["admin_level", "crop_name", "jurisdiction_name"]) +RAW_YIELDS = pandas.DataFrame.from_records( + [ + # CORN: 2016 must be excluded; mean(8, 10, 12, 10) over 2017-2020 == 10 + *( + ("PROVINCIAL", "USA008", "Delaware", "CORN", year, val) + for year, val in ( + (2016, 999.0), + (2017, 8.0), + (2018, 10.0), + (2019, 12.0), + (2020, 10.0), + ) + ), + # SOYBEANS: mean == 30 + *( + ("PROVINCIAL", "USA008", "Delaware", "SOYBEANS", year, 30.0) + for year in (2017, 2018, 2019, 2020) + ), + # (USA016, WHEAT): deliberately absent -> unmatched + ], + columns=[ + "admin_level", + "admin_id", + "jurisdiction_name", + "crop_name", + "year", + "yield_kg_per_ha", + ], +).set_index(["admin_level", "admin_id", "jurisdiction_name", "crop_name", "year"]) + + +def test_merge_emissions_and_yields() -> None: + result = merge_emissions_and_yields(emissions=EMISSIONS, raw_yields=RAW_YIELDS) + assert isinstance(result, pandas.DataFrame) + + iter_result = result.iterrows() + key, corn = next(iter_result) + assert key == ("PROVINCIAL", "CORN", "Delaware") + assert corn["yield_kg_per_ha"] == 10.0 # 4-year mean, 2016 excluded + assert corn["total_production_kg"] == 1000.0 # 100 ha × 10 + assert corn["emissions_factor_kgco2e_per_kg"] == 200.0 # 200 t × 1000 / 1000 kg + assert corn["peatland_occupation_fraction"] == 0.25 # 50 / 200 + + # zero production -> EF guarded to NaN (not inf); fraction still defined (0/80) + key, soy = next(iter_result) + assert key == ("PROVINCIAL", "SOYBEANS", "Delaware") + assert soy["total_production_kg"] == 0.0 + assert numpy.isnan(soy["emissions_factor_kgco2e_per_kg"]) + assert soy["peatland_occupation_fraction"] == 0.0 + + # unmatched yield -> NaN yield/production/EF; zero total_emissions -> NaN fraction + key, wheat = next(iter_result) + assert key == ("PROVINCIAL", "WHEAT", "Iowa") + assert numpy.isnan(wheat["yield_kg_per_ha"]) + assert numpy.isnan(wheat["total_production_kg"]) + assert numpy.isnan(wheat["emissions_factor_kgco2e_per_kg"]) + assert numpy.isnan(wheat["peatland_occupation_fraction"]) diff --git a/jdluc/__tests__/test_extract_county_fips.py b/jdluc/__tests__/test_extract_county_fips.py deleted file mode 100644 index 8b3a51b..0000000 --- a/jdluc/__tests__/test_extract_county_fips.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Unit tests for extract/county_fips.py. - -Mocks GEE so the tests run offline. Covers the (a) cache-hit / -force-true delete, (b) server-side paint recipe shape (FC filter, -fips_int property, paint property name), and (c) export call's GLAD -pinning + region-as-bounds invariants. -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import county_fips -from jdluc.utils.constants import ( - CONUS_STATE_FIPS, - GEE_COUNTY_FIPS_LABEL, - GLAD_CRS, - GLAD_CRS_TRANSFORM, -) - - -@pytest.fixture -def mock_io() -> Any: - """Patch the GEE-side IO helpers used by the extract module.""" - with ( - patch.object(county_fips, 'asset_exists') as asset_exists_mock, - patch.object(county_fips, 'delete_asset_if_present') as delete_mock, - patch.object(county_fips, 'wait_for_export_task') as wait_mock, - ): - ns = MagicMock() - ns.asset_exists = asset_exists_mock - ns.delete_asset = delete_mock - ns.wait_export = wait_mock - # By default the extract function is invoked on a cache miss - # (orchestrator already gated). Asset is present after export. - asset_exists_mock.return_value = True - yield ns - - -def test_extract_runs_paint_then_export_then_wait(mock_io: Any) -> None: - """Happy path: builds FC, builds image, starts export, waits, returns id.""" - with patch.object(county_fips, '_start_fips_export') as start_export: - export_task = MagicMock() - start_export.return_value = export_task - with ( - patch.object(county_fips, '_build_conus_counties_fc') as build_fc, - patch.object(county_fips, '_build_fips_image') as build_image, - ): - fc = MagicMock(name='counties_fc') - geom = MagicMock(name='geometry') - bbox = MagicMock(name='bbox') - fc.geometry.return_value = geom - geom.bounds.return_value = bbox - build_fc.return_value = fc - build_image.return_value = 'fips_image_sentinel' - - result = county_fips.extract_county_fips(gcp_project='ws-dev') - - assert result == GEE_COUNTY_FIPS_LABEL - # Force-false: no pre-delete on the target asset. - mock_io.delete_asset.assert_not_called() - # Image is built from the fc the helper produced. - build_image.assert_called_once_with(fc) - # Export call uses the painted image, the inventory asset id, and - # the FC's bounding rectangle. - start_export.assert_called_once() - kwargs = start_export.call_args.kwargs - assert kwargs['image'] == 'fips_image_sentinel' - assert kwargs['asset_id'] == GEE_COUNTY_FIPS_LABEL - assert kwargs['region'] is bbox - assert 'description' in kwargs - # Caller waits on the export. - mock_io.wait_export.assert_called_once_with(export_task, GEE_COUNTY_FIPS_LABEL) - - -def test_force_true_deletes_existing_asset_first(mock_io: Any) -> None: - """force=True clears the target before the new export runs.""" - with ( - patch.object(county_fips, '_start_fips_export') as start_export, - patch.object(county_fips, '_build_conus_counties_fc'), - patch.object(county_fips, '_build_fips_image'), - ): - start_export.return_value = MagicMock() - county_fips.extract_county_fips(gcp_project='ws-dev', force=True) - - mock_io.delete_asset.assert_called_once_with(GEE_COUNTY_FIPS_LABEL) - - -def test_extract_raises_if_post_export_asset_missing(mock_io: Any) -> None: - """Defensive: if the export 'completes' but no asset lands, fail loudly.""" - mock_io.asset_exists.return_value = False - with ( - patch.object(county_fips, '_start_fips_export', return_value=MagicMock()), - patch.object(county_fips, '_build_conus_counties_fc'), - patch.object(county_fips, '_build_fips_image'), - ): - with pytest.raises(RuntimeError, match='asset is not present'): - county_fips.extract_county_fips(gcp_project='ws-dev') - - -def test_build_conus_counties_fc_filters_and_maps() -> None: - """The server-side FC builder filters STATEFP and applies the tag mapper.""" - with patch('jdluc.extract.county_fips.ee') as ee_mock: - fc_root = MagicMock(name='fc_root') - filtered = MagicMock(name='filtered') - mapped = MagicMock(name='mapped') - ee_mock.FeatureCollection.return_value = fc_root - fc_root.filter.return_value = filtered - filtered.map.return_value = mapped - ee_mock.Filter.inList.return_value = 'inList_filter_sentinel' - - result = county_fips._build_conus_counties_fc() - - assert result is mapped - ee_mock.FeatureCollection.assert_called_once_with('TIGER/2018/Counties') - ee_mock.Filter.inList.assert_called_once_with('STATEFP', CONUS_STATE_FIPS) - fc_root.filter.assert_called_once_with('inList_filter_sentinel') - # The mapper passed to .map() is the tag-with-fips-int helper. - filtered.map.assert_called_once_with(county_fips._tag_with_fips_int) - - -def test_tag_with_fips_int_parses_geoid_to_int_and_sets_property() -> None: - """Per-feature tagger: GEOID string → ee.Number.parse → toInt → set.""" - with patch('jdluc.extract.county_fips.ee') as ee_mock: - feature = MagicMock(name='feature') - feature.get.return_value = 'geoid_value' - # Each step in the chain returns a distinct sentinel so we can - # verify the call sequence. - ee_mock.String.return_value = 'string_sentinel' - parsed = MagicMock(name='parsed_number') - ee_mock.Number.parse.return_value = parsed - parsed.toInt.return_value = 'int_sentinel' - feature.set.return_value = 'feature_with_fips_int' - - result = county_fips._tag_with_fips_int(feature) - - assert result == 'feature_with_fips_int' - feature.get.assert_called_once_with('GEOID') - ee_mock.String.assert_called_once_with('geoid_value') - ee_mock.Number.parse.assert_called_once_with('string_sentinel') - parsed.toInt.assert_called_once_with() - feature.set.assert_called_once_with('fips_int', 'int_sentinel') - - -def test_build_fips_image_paints_fips_int_and_self_masks() -> None: - """The image builder paints the fips_int property into a constant=0 int image.""" - with patch('jdluc.extract.county_fips.ee') as ee_mock: - constant = MagicMock(name='constant') - as_int = MagicMock(name='int') - painted = MagicMock(name='painted') - masked = MagicMock(name='masked') - renamed = MagicMock(name='renamed') - ee_mock.Image.constant.return_value = constant - constant.int.return_value = as_int - as_int.paint.return_value = painted - painted.selfMask.return_value = masked - masked.rename.return_value = renamed - - fc_sentinel = 'fc_sentinel' - result = county_fips._build_fips_image(fc_sentinel) - - assert result is renamed - ee_mock.Image.constant.assert_called_once_with(0) - constant.int.assert_called_once_with() - as_int.paint.assert_called_once_with(fc_sentinel, 'fips_int') - painted.selfMask.assert_called_once_with() - masked.rename.assert_called_once_with('county_fips') - # No inline .reproject() — the projection is pinned at the export - # sink instead, since inline reproject re-evaluates the polygon - # paint per tile (investigation doc § "Bisect probe: inline paint - # doesn't amortize, asset-loaded mask does"). - masked.reproject.assert_not_called() - - -def test_start_fips_export_pins_glad_grid_and_uses_supplied_region() -> None: - """Export targets the GLAD-pinned grid; region passes through unchanged.""" - with patch('jdluc.extract.county_fips.ee') as ee_mock: - export_task = MagicMock() - ee_mock.batch.Export.image.toAsset.return_value = export_task - - result = county_fips._start_fips_export( - image='img', - asset_id='target_asset', - region='region_sentinel', - description='extract_county_fips_smoke', - ) - - assert result is export_task - kwargs = ee_mock.batch.Export.image.toAsset.call_args.kwargs - assert kwargs['image'] == 'img' - assert kwargs['assetId'] == 'target_asset' - assert kwargs['region'] == 'region_sentinel' - assert kwargs['crs'] == GLAD_CRS - assert kwargs['crsTransform'] == GLAD_CRS_TRANSFORM - assert kwargs['description'] == 'extract_county_fips_smoke' - export_task.start.assert_called_once() diff --git a/jdluc/__tests__/test_extract_county_fips_integration.py b/jdluc/__tests__/test_extract_county_fips_integration.py deleted file mode 100644 index 0143500..0000000 --- a/jdluc/__tests__/test_extract_county_fips_integration.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Integration test for extract/county_fips.py at IA scale. - -Exercises the painting recipe against live GEE without doing the full -CONUS export. Filters TIGER 2018 counties to Iowa (FIPS 19), runs the -production helpers (`_tag_with_fips_int`, `_build_fips_image`) to build -the painted image, then samples each county at its centroid and -confirms the painted FIPS matches the county's GEOID. - -This is the "reproduces the expected IA mask" check for the painted -county-FIPS asset. Marked ``@pytest.mark.integration`` — needs GEE -credentials but no GCS staging. -""" - -from __future__ import annotations - -from typing import Any - -import ee -import pytest - -from jdluc.extract import county_fips -from jdluc.utils.constants import ( - GCP_PROJECT, - GEE_TIGER_COUNTIES, - GLAD_CRS, -) -from jdluc.utils.gee import initialize_gee - - -@pytest.fixture(scope='module') -def _ee_initialized() -> None: - initialize_gee(GCP_PROJECT) - - -@pytest.mark.integration -def test_paint_recipe_assigns_correct_fips_at_iowa_county_centroids( - _ee_initialized: None, -) -> None: - """Paint IA counties; sample at each centroid; FIPS should match GEOID.""" - iowa_counties = ( - ee.FeatureCollection(GEE_TIGER_COUNTIES) - .filter(ee.Filter.eq('STATEFP', '19')) - .map(county_fips._tag_with_fips_int) - ) - fips_image = county_fips._build_fips_image(iowa_counties) - - # Replace each county polygon with its centroid; the centroid is - # not guaranteed to be inside an irregular polygon, so we tolerate - # a small number of mismatches and only fail on a meaningful drift. - centroids = iowa_counties.map( - lambda f: f.setGeometry(f.geometry().centroid(maxError=10)) - ) - - sampled = fips_image.sampleRegions( - collection=centroids, - scale=30, - projection=GLAD_CRS, - geometries=False, - ).getInfo() - features = sampled.get('features', []) - - assert len(features) >= 90, ( - f'Expected ~99 Iowa counties; got {len(features)} features. ' - f'TIGER may have changed.' - ) - - mismatches: list[tuple[int, Any]] = [] - masked: list[int] = [] - for f in features: - props = f.get('properties', {}) - expected = int(props['GEOID']) - actual = props.get(county_fips.BAND_NAME) - if actual is None: - masked.append(expected) - elif int(actual) != expected: - mismatches.append((expected, actual)) - - assert not mismatches, ( - f'Painted FIPS disagrees with GEOID at {len(mismatches)} centroids: ' - f'{mismatches[:5]}' - ) - # Centroid-outside-polygon is a known artifact for ≤ a handful of - # irregular county shapes; reject only if the rate is unreasonable. - assert len(masked) <= 3, ( - f'{len(masked)} centroids fell outside their polygon (masked). ' - f'Expected ≤ 3 for IA: {masked}' - ) - - -@pytest.mark.integration -def test_paint_recipe_masks_pixels_outside_any_county( - _ee_initialized: None, -) -> None: - """Pixels outside any IA county are masked (selfMask drops the constant=0).""" - iowa_counties = ( - ee.FeatureCollection(GEE_TIGER_COUNTIES) - .filter(ee.Filter.eq('STATEFP', '19')) - .map(county_fips._tag_with_fips_int) - ) - fips_image = county_fips._build_fips_image(iowa_counties) - - # Pick a point clearly inside Lake Michigan (no county). Sampling - # there should yield a masked / null value. - offshore_point = ee.Geometry.Point([-87.0, 43.5]) - sampled = fips_image.sampleRegions( - collection=ee.FeatureCollection([ee.Feature(offshore_point)]), - scale=30, - projection=GLAD_CRS, - geometries=False, - ).getInfo() - features = sampled.get('features', []) - # sampleRegions drops features that fall on masked pixels — so the - # output collection should be empty for a fully-masked sample point. - assert features == [], f'Expected offshore point to be masked; got {features}' - - -@pytest.mark.integration -def test_paint_recipe_pixel_value_at_known_polk_county_point( - _ee_initialized: None, -) -> None: - """A known IA point gets the right county FIPS at GLAD 30 m. - - Independent of the centroid-sweep test above: anchors the recipe - against a hand-chosen point with a known answer. Picks Des Moines - (~ -93.6, 41.6) which is squarely inside Polk County, IA - (GEOID=19153). - """ - iowa_counties = ( - ee.FeatureCollection(GEE_TIGER_COUNTIES) - .filter(ee.Filter.eq('STATEFP', '19')) - .map(county_fips._tag_with_fips_int) - ) - fips_image = county_fips._build_fips_image(iowa_counties) - - point = ee.Geometry.Point([-93.6, 41.6]) - sampled = fips_image.sampleRegions( - collection=ee.FeatureCollection([ee.Feature(point)]), - scale=30, - projection=GLAD_CRS, - geometries=False, - ).getInfo() - features = sampled.get('features', []) - assert ( - len(features) == 1 - ), f'Expected 1 sample at the known IA point; got {features}' - fips = features[0]['properties'].get(county_fips.BAND_NAME) - assert ( - fips == 19153 - ), f'Expected Polk County, IA (FIPS 19153) at (-93.6, 41.6); got {fips}' diff --git a/jdluc/__tests__/test_extract_extract.py b/jdluc/__tests__/test_extract_extract.py deleted file mode 100644 index 48ac685..0000000 --- a/jdluc/__tests__/test_extract_extract.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Unit tests for extract/extract.py orchestrator. - -Covers the cache-hit / cache-miss / failure-capture paths using the -injectable ``extractor_factory`` and ``asset_exists`` hooks — no GEE, -no HTTP, no per-dataset modules required. -""" - -from collections.abc import Callable - -import pytest - -from jdluc.extract.extract import ( - ExtractError, - ExtractorCallable, - ExtractResult, - extract_all, -) -from jdluc.utils.constants import ( - DATASET_INVENTORY, - NON_NATIVE_DATASETS, -) - - -def _build_factory( - produced: dict[str, str] | None = None, - raising: dict[str, Exception] | None = None, - calls: list[tuple[str, str, bool]] | None = None, -) -> Callable[[str], ExtractorCallable]: - """Build a fake ``extractor_factory`` for a given dispatch plan. - - - ``produced[family]`` → the asset ID the extractor should return. - - ``raising[family]`` → an exception to raise instead. - - ``calls`` — if provided, each invocation appends ``(family, gcp, force)``. - """ - produced = produced or {} - raising = raising or {} - - def factory(family: str) -> ExtractorCallable: - def extractor(gcp_project: str, force: bool = False) -> str: - if calls is not None: - calls.append((family, gcp_project, force)) - if family in raising: - raise raising[family] - return produced[family] - - return extractor - - return factory - - -def _all_present(_: str) -> bool: - return True - - -def _none_present(_: str) -> bool: - return False - - -def test_every_asset_cached_skips_all_extractors() -> None: - calls: list[tuple[str, str, bool]] = [] - factory = _build_factory(calls=calls) - - result = extract_all( - gcp_project='ws-dev', - extractor_factory=factory, - asset_exists=_all_present, - ) - - assert result.cached == NON_NATIVE_DATASETS - assert result.extracted == [] - assert result.failed == {} - assert calls == [] - - -def test_missing_assets_trigger_extractor_dispatch() -> None: - produced = { - family: DATASET_INVENTORY[family]['gee_asset_id'] - for family in NON_NATIVE_DATASETS - } - calls: list[tuple[str, str, bool]] = [] - factory = _build_factory(produced=produced, calls=calls) - - result = extract_all( - gcp_project='ws-dev', - extractor_factory=factory, - asset_exists=_none_present, - ) - - assert result.cached == [] - assert result.extracted == NON_NATIVE_DATASETS - assert result.failed == {} - # Order of `calls` is non-deterministic under the parallel orchestrator; - # only assert membership and per-call args. - assert sorted(c[0] for c in calls) == sorted(NON_NATIVE_DATASETS) - # force propagates as False by default. - assert all(c[2] is False for c in calls) - - -def test_force_true_bypasses_cache_for_all_datasets() -> None: - produced = { - family: DATASET_INVENTORY[family]['gee_asset_id'] - for family in NON_NATIVE_DATASETS - } - calls: list[tuple[str, str, bool]] = [] - factory = _build_factory(produced=produced, calls=calls) - - result = extract_all( - gcp_project='ws-dev', - force=True, - extractor_factory=factory, - asset_exists=_all_present, # even if cached, force=True still re-runs - ) - - assert result.cached == [] - assert result.extracted == NON_NATIVE_DATASETS - assert all(c[2] is True for c in calls) - - -def test_failed_extractor_is_recorded_and_other_datasets_continue() -> None: - produced = { - family: DATASET_INVENTORY[family]['gee_asset_id'] - for family in NON_NATIVE_DATASETS - if family != 'huang_bgb' - } - raising: dict[str, Exception] = {'huang_bgb': RuntimeError('Figshare 503')} - factory = _build_factory(produced=produced, raising=raising) - - result = extract_all( - gcp_project='ws-dev', - extractor_factory=factory, - asset_exists=_none_present, - ) - - assert 'huang_bgb' in result.failed - assert 'Figshare 503' in result.failed['huang_bgb'] - expected_extracted = [f for f in NON_NATIVE_DATASETS if f != 'huang_bgb'] - assert result.extracted == expected_extracted - assert result.cached == [] - - -def test_extract_error_summarizes_failures() -> None: - partial = ExtractResult( - cached=['ipcc_climate_zones'], - extracted=['huang_bgb'], - failed={'nass_yields': 'HTTP 500', 'harris_agb': 'GCS permission denied'}, - ) - err = ExtractError(partial) - msg = str(err) - assert '2 dataset' in msg - assert 'nass_yields' in msg - assert 'harris_agb' in msg - assert err.result is partial - - -def test_extractor_returning_wrong_asset_id_still_counts_as_extracted( - caplog: pytest.LogCaptureFixture, -) -> None: - # Extractor returns a path that doesn't match DATASET_INVENTORY — the - # orchestrator warns but doesn't fail. Keeps the contract liberal for - # now; a stricter check would be a separate enforcement step. - produced = { - family: DATASET_INVENTORY[family]['gee_asset_id'] - for family in NON_NATIVE_DATASETS - } - produced['ipcc_climate_zones'] = 'gee/asset/path' - factory = _build_factory(produced=produced) - - with caplog.at_level('WARNING', logger='jdluc.extract.extract'): - result = extract_all( - gcp_project='ws-dev', - extractor_factory=factory, - asset_exists=_none_present, - ) - - assert 'ipcc_climate_zones' in result.extracted - assert any('inventory expects' in r.message for r in caplog.records) - - -def test_extractors_run_in_parallel() -> None: - """Cold-cache wall-clock should be closer to max(per-extractor) than sum. - - Each fake extractor sleeps 0.1s. Sequential execution would take - ~0.6s for 6 extractors; the parallel orchestrator should finish in - well under 0.3s (5x headroom over the per-extractor sleep). - """ - import time - - produced = { - family: DATASET_INVENTORY[family]['gee_asset_id'] - for family in NON_NATIVE_DATASETS - } - - def factory(family: str) -> ExtractorCallable: - def extractor(gcp_project: str, force: bool = False) -> str: - time.sleep(0.1) - return produced[family] - - return extractor - - start = time.monotonic() - result = extract_all( - gcp_project='ws-dev', - extractor_factory=factory, - asset_exists=_none_present, - ) - elapsed = time.monotonic() - start - - assert result.extracted == NON_NATIVE_DATASETS - assert result.failed == {} - # Sequential floor is 0.6s; parallel ceiling is closer to 0.1s. Use - # 0.3s as a generous bound that still proves concurrency. - assert elapsed < 0.3, f'extract_all took {elapsed:.3f}s; expected parallel run' diff --git a/jdluc/__tests__/test_extract_gfw_peatlands.py b/jdluc/__tests__/test_extract_gfw_peatlands.py deleted file mode 100644 index d9e5889..0000000 --- a/jdluc/__tests__/test_extract_gfw_peatlands.py +++ /dev/null @@ -1,359 +0,0 @@ -"""Unit tests for extract/gfw_peatlands.py. - -Mocks GCS and GEE ingestion so the tests run offline. Validates URL -construction (no HTTP probe — the URL is template-constructed), -per-tile dispatch, the force-rebuild path, the cache-hit path (all -19 tiles already ingested), and a custom tile-list override. - -The GFW Peatlands module is structurally a clone of extract/harris_agb.py -under the GFW Global Peatlands raster swap. Tests mirror the -test_extract_harris_agb.py shape with two key differences: - - URL construction is done from a template + API key (no - FeatureServer query), so no HTTP mocks are needed for resolution. - - The tile list is GFW-Peatlands-specific: GFW omits tiles with no - peatland pixels, so CONUS coverage is 19 tiles, not Harris - AGB's 21. -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import _tile_pipeline, gfw_peatlands -from jdluc.utils.constants import ( - GEE_GFW_PEATLANDS, - GFW_DATA_API_KEY, - GFW_PEATLANDS_URL_TEMPLATE, -) - - -@pytest.fixture -def mock_io() -> Any: - """Patch every IO helper used by the shared tile orchestrator. - - Submit and poll halves of ingestion are patched separately to match - the fan-out shape; ``submit`` returns a stable task ID per - asset_id so wait/cleanup bookkeeping is deterministic in assertions. - Patches land on ``_tile_pipeline`` (the orchestrator's import - surface) since the per-dataset module no longer references these - helpers directly. - """ - with ( - patch.object(_tile_pipeline, 'asset_exists') as asset_exists, - patch.object( - _tile_pipeline, 'create_image_collection_if_absent' - ) as create_coll, - patch.object(_tile_pipeline, 'delete_asset_if_present') as delete_asset, - patch.object(_tile_pipeline, 'delete_gcs_blob') as delete_gcs, - patch.object(_tile_pipeline, 'fetch_with_mirror') as fetch, - patch.object(_tile_pipeline, 'start_ingestion_no_wait') as submit, - patch.object(_tile_pipeline, 'upload_to_gcs') as upload, - patch.object(_tile_pipeline, 'wait_for_tasks') as poll, - ): - ns = MagicMock() - ns.asset_exists = asset_exists - ns.create_coll = create_coll - ns.delete_asset = delete_asset - ns.delete_gcs = delete_gcs - ns.fetch = fetch - ns.submit = submit - ns.upload = upload - ns.poll = poll - - upload.side_effect = lambda project, bucket, blob, path: f'gs://{bucket}/{blob}' - submit.side_effect = lambda gcs_uri, asset_id, **_kw: f'task_{asset_id}' - poll.return_value = {} - yield ns - - -# --------------------------------------------------------------------------- -# URL construction -# --------------------------------------------------------------------------- - - -def test_resolve_tile_url_uses_template_and_api_key() -> None: - url = gfw_peatlands._resolve_tile_url('40N_080W') - # URL incorporates the dataset version, tile_id, pixel_meaning, and key. - assert 'gfw_peatlands' in url - assert 'v20230315' in url - assert 'tile_id=40N_080W' in url - assert 'pixel_meaning=is' in url - assert f'x-api-key={GFW_DATA_API_KEY}' in url - - -def test_resolve_tile_url_template_round_trip() -> None: - """The constants module's URL template formats round-trip cleanly.""" - expected = GFW_PEATLANDS_URL_TEMPLATE.format( - tile_id='30N_090W', api_key=GFW_DATA_API_KEY - ) - assert gfw_peatlands._resolve_tile_url('30N_090W') == expected - - -def test_tile_image_asset_id_format() -> None: - aid = _tile_pipeline._tile_image_asset_id(GEE_GFW_PEATLANDS, '40N_080W') - assert aid == f'{GEE_GFW_PEATLANDS}/tile_40N_080W' - - -def test_conus_tile_ids_match_gfw_manifest() -> None: - """CONUS_TILE_IDS lists the 19 CONUS tiles GFW actually publishes. - - GFW's v20230315 manifest omits tiles with no peatland pixels. - The Harris AGB grid has 21 CONUS tiles; GFW publishes 19. - 30N_070W (Caribbean) and 30N_130W (Pacific) are absent — both - have no land at the corresponding latitudes. - """ - assert len(gfw_peatlands.CONUS_TILE_IDS) == 19 - assert '30N_070W' not in gfw_peatlands.CONUS_TILE_IDS - assert '30N_130W' not in gfw_peatlands.CONUS_TILE_IDS - # Spot-check land-bearing tiles are included. - for expected in ('40N_080W', '40N_090W', '50N_100W', '30N_080W'): - assert expected in gfw_peatlands.CONUS_TILE_IDS - - -# --------------------------------------------------------------------------- -# Main extract flow -# --------------------------------------------------------------------------- - - -def test_extract_gfw_peatlands_all_tiles_cache_hit_skips_download( - mock_io: Any, -) -> None: - """Every tile asset already exists → no fetch / upload / ingest.""" - mock_io.asset_exists.return_value = True - - asset_id = gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - assert asset_id == GEE_GFW_PEATLANDS - mock_io.fetch.assert_not_called() - mock_io.upload.assert_not_called() - mock_io.submit.assert_not_called() - mock_io.poll.assert_not_called() - mock_io.create_coll.assert_called_once_with(GEE_GFW_PEATLANDS) - - -def test_extract_gfw_peatlands_missing_tiles_are_ingested(mock_io: Any) -> None: - """Cache misses trigger fetch + upload + submit per tile, single bulk poll.""" - mock_io.asset_exists.return_value = False - - gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - # Phase A staging runs in a ThreadPoolExecutor, so call ordering is - # non-deterministic — assertions are unordered. - assert mock_io.fetch.call_count == 2 - fetch_kwargs_list = [c.kwargs for c in mock_io.fetch.call_args_list] - assert {kw['dataset'] for kw in fetch_kwargs_list} == {'gfw_peatlands'} - assert {kw['filename'] for kw in fetch_kwargs_list} == { - '40N_080W.tif', - '30N_090W.tif', - } - for kw in fetch_kwargs_list: - assert callable(kw['source']) - - assert mock_io.upload.call_count == 2 - assert mock_io.submit.call_count == 2 - submit_asset_ids = {call.args[1] for call in mock_io.submit.call_args_list} - assert submit_asset_ids == { - f'{GEE_GFW_PEATLANDS}/tile_40N_080W', - f'{GEE_GFW_PEATLANDS}/tile_30N_090W', - } - # Phase C is one bulk poll over both task IDs. - assert mock_io.poll.call_count == 1 - polled_task_ids = set(mock_io.poll.call_args.args[0]) - assert polled_task_ids == { - f'task_{GEE_GFW_PEATLANDS}/tile_40N_080W', - f'task_{GEE_GFW_PEATLANDS}/tile_30N_090W', - } - # GCS staging blobs are cleaned up after the run. - assert mock_io.delete_gcs.call_count == 2 - # Band name is passed through to submission. - for call in mock_io.submit.call_args_list: - assert call.kwargs['band_name'] == 'is_peatland' - - -def test_extract_gfw_peatlands_tile_url_resolver_returns_template_url( - mock_io: Any, -) -> None: - """The source callable formats the GFW URL template — no HTTP probe.""" - mock_io.asset_exists.return_value = False - captured_sources: list[Any] = [] - - def _capture( - local_path: str, - *, - dataset: str, - filename: str, - gcp_project: str, - source: Any, - **_kwargs: Any, - ) -> None: - del local_path, dataset, filename, gcp_project - captured_sources.append(source) - - mock_io.fetch.side_effect = _capture - - gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W'], - ) - resolved = captured_sources[0]() - expected = gfw_peatlands._resolve_tile_url('40N_080W') - assert resolved == expected - assert 'tile_id=40N_080W' in resolved - - -def test_extract_gfw_peatlands_force_with_subset_only_deletes_those_tiles( - mock_io: Any, -) -> None: - """force=True with explicit tile_ids deletes only those tiles, not the collection. - - Regression test mirrored from harris_agb: subset force-rebuilds must - not destroy sibling tiles in the parent collection. - """ - mock_io.asset_exists.return_value = False - - with patch.object(_tile_pipeline, '_delete_collection_if_present') as clear: - gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - force=True, - tile_ids=['40N_080W'], - ) - - clear.assert_not_called() - delete_targets = [c.args[0] for c in mock_io.delete_asset.call_args_list] - assert f'{GEE_GFW_PEATLANDS}/tile_40N_080W' in delete_targets - for sibling in ('30N_090W', '50N_100W'): - assert f'{GEE_GFW_PEATLANDS}/tile_{sibling}' not in delete_targets - assert mock_io.submit.call_args.kwargs['allow_overwrite'] is True - - -def test_extract_gfw_peatlands_force_with_default_tile_ids_clears_collection( - mock_io: Any, -) -> None: - """force=True with the default tile list drops the parent collection.""" - mock_io.asset_exists.return_value = True - - with patch.object(_tile_pipeline, '_delete_collection_if_present') as clear: - gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - force=True, - ) - - clear.assert_called_once() - - -def test_extract_gfw_peatlands_uses_default_conus_tile_list_when_none( - mock_io: Any, -) -> None: - """Default tile_ids=None resolves to CONUS_TILE_IDS (19 tiles).""" - mock_io.asset_exists.return_value = True # all cache-hit so no real work - gfw_peatlands.extract_gfw_peatlands(gcp_project='ws-dev') - # 19 cache-hit log paths, no fetches. - mock_io.fetch.assert_not_called() - # asset_exists was probed once per tile. - assert mock_io.asset_exists.call_count >= len(gfw_peatlands.CONUS_TILE_IDS) - - -def test_extract_gfw_peatlands_custom_tile_list_processes_only_given_tiles( - mock_io: Any, -) -> None: - """Custom tile_ids override; any other tile IDs are not touched.""" - mock_io.asset_exists.return_value = False - gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_100W', '50N_120W', '30N_080W'], - ) - - assert mock_io.fetch.call_count == 3 - # Phase A staging is non-deterministic, compare as a set. - fetch_filenames = {c.kwargs['filename'] for c in mock_io.fetch.call_args_list} - assert fetch_filenames == {'40N_100W.tif', '50N_120W.tif', '30N_080W.tif'} - - -# --------------------------------------------------------------------------- -# Failure & cleanup contracts -# --------------------------------------------------------------------------- - - -def test_staging_failure_does_not_leave_gcs_blob_behind(mock_io: Any) -> None: - """A Phase A staging exception is logged and isolated to that tile. - - With the only tile failing, there's nothing to submit and no blob to - clean up. The orchestrator returns the (empty) collection asset ID - rather than aborting — sibling tiles in a multi-tile run would still - complete. - """ - mock_io.asset_exists.return_value = False - mock_io.fetch.side_effect = RuntimeError('simulated download failure') - - asset_id = gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W'], - ) - assert asset_id == GEE_GFW_PEATLANDS - - mock_io.upload.assert_not_called() - mock_io.submit.assert_not_called() - mock_io.delete_gcs.assert_not_called() - - -def test_partial_ingest_failure_isolated(mock_io: Any) -> None: - """A FAILED ingest task does not abort siblings; cleanup still happens.""" - mock_io.asset_exists.return_value = False - mock_io.poll.return_value = { - f'task_{GEE_GFW_PEATLANDS}/tile_40N_080W': 'manifest invalid', - } - - asset_id = gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - assert asset_id == GEE_GFW_PEATLANDS - # Both blobs still got cleaned up — partial failure shouldn't leak GCS. - assert mock_io.delete_gcs.call_count == 2 - - -def test_submit_exception_still_cleans_up_orphan_blob(mock_io: Any) -> None: - """A Phase B submission exception still cleans the staged GCS blob. - - The blob made it to GCS during Phase A but no task ID was issued, so - the orchestrator records it in ``orphan_blobs`` for Phase D cleanup. - """ - mock_io.asset_exists.return_value = False - mock_io.submit.side_effect = RuntimeError('GEE 500 startIngestion') - - asset_id = gfw_peatlands.extract_gfw_peatlands( - gcp_project='ws-dev', - tile_ids=['40N_080W'], - ) - assert asset_id == GEE_GFW_PEATLANDS - - mock_io.upload.assert_called_once() - # Submission failed — no task IDs to poll. - mock_io.poll.assert_not_called() - # Blob still gets cleaned up via orphan_blobs. - mock_io.delete_gcs.assert_called_once() - - -# --------------------------------------------------------------------------- -# Regression guard: no shapefile / vector libs in the import graph -# --------------------------------------------------------------------------- - - -def test_no_shapefile_or_shapely_imports() -> None: - """No shapefile/shapely imports — this is a raster ingest, not a vector one.""" - import sys - - sys.modules.pop('jdluc.extract.gfw_peatlands', None) - import jdluc.extract.gfw_peatlands as fresh # noqa: F401 - - assert 'shapefile' not in sys.modules # pyshp - assert 'shapely' not in sys.modules - assert 'geopandas' not in sys.modules diff --git a/jdluc/__tests__/test_extract_harris_agb.py b/jdluc/__tests__/test_extract_harris_agb.py deleted file mode 100644 index f192b18..0000000 --- a/jdluc/__tests__/test_extract_harris_agb.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Unit tests for extract/harris_agb.py. - -Mocks HTTP, GCS, and GEE ingestion so the tests run offline. Validates -URL construction, per-tile dispatch, the force-rebuild path, and the -cache-hit path (all 19 tiles already ingested). -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import _tile_pipeline, harris_agb -from jdluc.utils.constants import ( - GEE_HARRIS_AGB, - HARRIS_AGB_ARCGIS_FEATURESERVER, -) - - -@pytest.fixture -def mock_io() -> Any: - """Patch every _io helper used by the shared tile orchestrator. - - The submit (``start_ingestion_no_wait``) and poll (``wait_for_tasks``) - halves of ingestion are patched separately — together they replace - the pre-Phase-9 single ``start_ingestion_and_wait`` call. ``submit`` - returns a stable task ID per asset_id so wait/cleanup bookkeeping is - deterministic in assertions. Patches land on ``_tile_pipeline`` (the - orchestrator's import surface) since the per-dataset module no - longer references these helpers directly. - """ - with ( - patch.object(_tile_pipeline, 'asset_exists') as asset_exists, - patch.object( - _tile_pipeline, 'create_image_collection_if_absent' - ) as create_coll, - patch.object(_tile_pipeline, 'delete_asset_if_present') as delete_asset, - patch.object(_tile_pipeline, 'delete_gcs_blob') as delete_gcs, - patch.object(_tile_pipeline, 'fetch_with_mirror') as fetch, - patch.object(_tile_pipeline, 'start_ingestion_no_wait') as submit, - patch.object(_tile_pipeline, 'upload_to_gcs') as upload, - patch.object(_tile_pipeline, 'wait_for_tasks') as poll, - ): - - ns = MagicMock() - ns.asset_exists = asset_exists - ns.create_coll = create_coll - ns.delete_asset = delete_asset - ns.delete_gcs = delete_gcs - ns.fetch = fetch - ns.submit = submit - ns.upload = upload - ns.poll = poll - - upload.side_effect = lambda project, bucket, blob, path: f'gs://{bucket}/{blob}' - # Default: every submission returns task_; every poll - # reports zero failures. - submit.side_effect = lambda gcs_uri, asset_id, **_kw: f'task_{asset_id}' - poll.return_value = {} - yield ns - - -def _fake_feature_server_response(tile_id: str) -> Any: - resp = MagicMock() - resp.raise_for_status = MagicMock() - resp.json = MagicMock( - return_value={ - 'features': [ - { - 'attributes': { - 'tile_id': tile_id, - 'Mg_ha_1_download': f'https://signed/{tile_id}.tif', - } - } - ] - } - ) - return resp - - -def test_conus_tile_ids_match_feature_server_manifest() -> None: - # Full 3×7 grid would be 21 tiles, but the FeatureServer omits - # 30N_070W (Caribbean) and 30N_130W (Pacific) — both ocean with - # no biomass — so we list only the 19 tiles it publishes. - assert len(harris_agb.CONUS_TILE_IDS) == 19 - assert len(set(harris_agb.CONUS_TILE_IDS)) == 19 - assert '30N_070W' not in harris_agb.CONUS_TILE_IDS - assert '30N_130W' not in harris_agb.CONUS_TILE_IDS - for expected in ('40N_080W', '40N_090W', '50N_100W', '30N_080W'): - assert expected in harris_agb.CONUS_TILE_IDS - - -def test_feature_server_url_resolution() -> None: - tile_id = '40N_080W' - with patch('jdluc.extract.harris_agb.requests.get') as get: - get.return_value = _fake_feature_server_response(tile_id) - url = harris_agb._fetch_tile_download_url(tile_id) - - assert url == f'https://signed/{tile_id}.tif' - call = get.call_args - assert call.args[0] == HARRIS_AGB_ARCGIS_FEATURESERVER - assert call.kwargs['params']['where'] == f"tile_id='{tile_id}'" - assert 'Mg_ha_1_download' in call.kwargs['params']['outFields'] - - -def test_feature_server_missing_tile_raises() -> None: - resp = MagicMock() - resp.raise_for_status = MagicMock() - resp.json = MagicMock(return_value={'features': []}) - with patch('jdluc.extract.harris_agb.requests.get', return_value=resp): - with pytest.raises(RuntimeError, match='not found on FeatureServer'): - harris_agb._fetch_tile_download_url('99N_999W') - - -def test_extract_harris_agb_all_tiles_cache_hit_skips_download(mock_io: Any) -> None: - mock_io.asset_exists.return_value = True - - with patch('jdluc.extract.harris_agb.requests.get') as get: - asset_id = harris_agb.extract_harris_agb( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - assert asset_id == GEE_HARRIS_AGB - get.assert_not_called() - mock_io.fetch.assert_not_called() - mock_io.upload.assert_not_called() - mock_io.submit.assert_not_called() - mock_io.poll.assert_not_called() - mock_io.create_coll.assert_called_once_with(GEE_HARRIS_AGB) - - -def test_extract_harris_agb_missing_tiles_are_ingested(mock_io: Any) -> None: - mock_io.asset_exists.return_value = False - - harris_agb.extract_harris_agb( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - # Both tiles got fetched via the mirror helper + uploaded + submitted. - # Phase A staging runs in a ThreadPoolExecutor, so call ordering is - # non-deterministic — assertions are unordered. - assert mock_io.fetch.call_count == 2 - fetch_kwargs_list = [c.kwargs for c in mock_io.fetch.call_args_list] - assert {kw['dataset'] for kw in fetch_kwargs_list} == {'harris_agb'} - assert {kw['filename'] for kw in fetch_kwargs_list} == { - '40N_080W.tif', - '30N_090W.tif', - } - # The source is a callable (deferred FeatureServer resolver), not a bare string. - for kw in fetch_kwargs_list: - assert callable(kw['source']) - - assert mock_io.upload.call_count == 2 - # Phase B submit fans out one task per staged tile. - assert mock_io.submit.call_count == 2 - submit_asset_ids = {call.args[1] for call in mock_io.submit.call_args_list} - assert submit_asset_ids == { - f'{GEE_HARRIS_AGB}/tile_40N_080W', - f'{GEE_HARRIS_AGB}/tile_30N_090W', - } - # Phase C is one bulk poll over both task IDs. - assert mock_io.poll.call_count == 1 - polled_task_ids = set(mock_io.poll.call_args.args[0]) - assert polled_task_ids == { - f'task_{GEE_HARRIS_AGB}/tile_40N_080W', - f'task_{GEE_HARRIS_AGB}/tile_30N_090W', - } - # Phase D cleans up every staged blob. - assert mock_io.delete_gcs.call_count == 2 - - -def test_extract_harris_agb_tile_url_resolver_uses_feature_server(mock_io: Any) -> None: - """The source callable passed to fetch_with_mirror hits the FeatureServer.""" - mock_io.asset_exists.return_value = False - captured_sources: list[Any] = [] - - def _capture( - local_path: str, - *, - dataset: str, - filename: str, - gcp_project: str, - source: Any, - **_kwargs: Any, - ) -> None: - del local_path, dataset, filename, gcp_project - captured_sources.append(source) - - mock_io.fetch.side_effect = _capture - - with patch('jdluc.extract.harris_agb.requests.get') as get: - get.return_value = _fake_feature_server_response('40N_080W') - harris_agb.extract_harris_agb( - gcp_project='ws-dev', - tile_ids=['40N_080W'], - ) - # Invoking the resolver triggers the FeatureServer request. - resolved = captured_sources[0]() - - assert resolved == 'https://signed/40N_080W.tif' - assert get.call_count == 1 - - -def test_extract_harris_agb_force_with_subset_only_deletes_those_tiles( - mock_io: Any, -) -> None: - """force=True with explicit tile_ids deletes only those tiles, not the collection. - - Regression test: force=True historically triggered - _delete_collection_if_present() which iterated CONUS_TILE_IDS, - silently destroying any tile assets outside the caller's tile_ids - list. - """ - mock_io.asset_exists.return_value = False - - with patch.object(_tile_pipeline, '_delete_collection_if_present') as clear: - harris_agb.extract_harris_agb( - gcp_project='ws-dev', - force=True, - tile_ids=['40N_080W'], - ) - - clear.assert_not_called() - # The single tile asset was pre-deleted before re-extraction. - delete_targets = [c.args[0] for c in mock_io.delete_asset.call_args_list] - assert f'{GEE_HARRIS_AGB}/tile_40N_080W' in delete_targets - # No sibling tiles touched. - for sibling in ('30N_090W', '50N_100W'): - assert f'{GEE_HARRIS_AGB}/tile_{sibling}' not in delete_targets - # force → submission runs with allow_overwrite=True. - assert mock_io.submit.call_args.kwargs['allow_overwrite'] is True - - -def test_extract_harris_agb_force_with_default_tile_ids_clears_collection( - mock_io: Any, -) -> None: - """force=True with the default tile list (None → CONUS) drops the collection. - - Pairs with the subset-only-deletes-those-tiles regression test: - full-rebuild semantics keep the existing nuke-and-recreate behavior - so partial collections from prior crashes get cleaned up. - """ - mock_io.asset_exists.return_value = True - - with patch.object(_tile_pipeline, '_delete_collection_if_present') as clear: - harris_agb.extract_harris_agb( - gcp_project='ws-dev', - force=True, - # tile_ids=None → defaults to CONUS_TILE_IDS - ) - - clear.assert_called_once() - - -def test_extract_harris_agb_partial_ingest_failure_isolated(mock_io: Any) -> None: - """One FAILED task does not abort siblings; orchestrator returns the IC.""" - mock_io.asset_exists.return_value = False - mock_io.poll.return_value = { - f'task_{GEE_HARRIS_AGB}/tile_40N_080W': 'manifest invalid', - } - - asset_id = harris_agb.extract_harris_agb( - gcp_project='ws-dev', - tile_ids=['40N_080W', '30N_090W'], - ) - - assert asset_id == GEE_HARRIS_AGB - # Both blobs still got cleaned up — partial failure shouldn't leak GCS. - assert mock_io.delete_gcs.call_count == 2 - - -def test_extract_harris_agb_staging_failure_does_not_leave_gcs_blob( - mock_io: Any, -) -> None: - """A Phase A staging exception aborts that tile's submit — no blob, no leak.""" - mock_io.asset_exists.return_value = False - mock_io.fetch.side_effect = RuntimeError('simulated download failure') - - # The orchestrator logs and continues — staging exceptions don't abort - # the run, but with the only tile failing there's nothing left to submit. - asset_id = harris_agb.extract_harris_agb( - gcp_project='ws-dev', - tile_ids=['40N_080W'], - ) - - assert asset_id == GEE_HARRIS_AGB - # Failed staging never reached upload, so there's no blob to delete. - mock_io.upload.assert_not_called() - mock_io.submit.assert_not_called() - mock_io.delete_gcs.assert_not_called() diff --git a/jdluc/__tests__/test_extract_harris_agb_integration.py b/jdluc/__tests__/test_extract_harris_agb_integration.py deleted file mode 100644 index 56ccd74..0000000 --- a/jdluc/__tests__/test_extract_harris_agb_integration.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Integration tests for the fan-out shape of extract/harris_agb.py. - -Marked ``@pytest.mark.integration`` — requires live GEE credentials and -write access to the GCS staging bucket. Single-tile coverage is enough -to validate the orchestrator structurally; CONUS-scale wall-time validation -is a separate manual verification step. - -Tests are DESTRUCTIVE against the per-tile asset (delete + rebuild) but -leave the rest of the collection untouched. They run with ``force=True`` -so a parent ``ImageCollection`` and unrelated tile siblings are preserved. -""" - -import pytest - -from jdluc.extract import _tile_pipeline, harris_agb -from jdluc.extract.harris_agb import ( - GCS_STAGING_PREFIX, - extract_harris_agb, -) -from jdluc.utils.constants import ( - GCP_PROJECT, - GCS_BUCKET_NAME, - GEE_HARRIS_AGB, -) -from jdluc.utils.gee import ( - asset_exists, - delete_asset_if_present, - initialize_gee, -) - -# 40N_080W is the Delaware-containing tile; smaller payload than the -# heavier 50N_* tiles to the north, so this is the cheapest single-tile -# round trip for validating orchestrator structure. -_TEST_TILE_ID: str = '40N_080W' - - -@pytest.fixture(scope='module') -def _ee_initialized() -> None: - initialize_gee(GCP_PROJECT) - - -def _list_staging_blobs(run_id_prefix: str = '') -> list[str]: - """Return the names of any GCS staging blobs under the harris_agb prefix. - - Used by the cleanup-leak check; ``run_id_prefix`` filters to a single - run when set. - """ - from google.cloud import storage - - client = storage.Client(project=GCP_PROJECT) - prefix = ( - f'{GCS_STAGING_PREFIX}/{run_id_prefix}' if run_id_prefix else GCS_STAGING_PREFIX - ) - return [b.name for b in client.list_blobs(GCS_BUCKET_NAME, prefix=prefix)] - - -@pytest.mark.integration -def test_harris_agb_single_tile_fan_out_round_trip(_ee_initialized: None) -> None: - """Single-tile clean-cache extract round-trips through the fan-out path.""" - tile_asset_id = _tile_pipeline._tile_image_asset_id(GEE_HARRIS_AGB, _TEST_TILE_ID) - - # DESTRUCTIVE: tear down just this tile, leaving sibling tiles untouched. - delete_asset_if_present(tile_asset_id) - - asset_id = extract_harris_agb( - gcp_project=GCP_PROJECT, - force=True, - tile_ids=[_TEST_TILE_ID], - ) - - assert asset_id == GEE_HARRIS_AGB - assert asset_exists( - tile_asset_id - ), f'fan-out extract did not materialize {tile_asset_id}' - - -@pytest.mark.integration -def test_harris_agb_no_gcs_staging_objects_left_behind(_ee_initialized: None) -> None: - """After a clean-cache extract, the GCS staging prefix is empty. - - Phase D defers cleanup to end-of-run; this test asserts that nothing - is left in the bucket after the prior round-trip test completes. - Depends on the previous test having run and seeded the asset. - """ - leftover = _list_staging_blobs() - assert leftover == [], ( - f'GCS staging objects leaked under ' - f'gs://{GCS_BUCKET_NAME}/{GCS_STAGING_PREFIX}/: {leftover}' - ) - - -def test_harris_agb_integration_module_imports_cleanly() -> None: - """Non-integration sanity check — catches import-time regressions.""" - assert callable(extract_harris_agb) - assert _TEST_TILE_ID in harris_agb.CONUS_TILE_IDS diff --git a/jdluc/__tests__/test_extract_huang_bgb.py b/jdluc/__tests__/test_extract_huang_bgb.py deleted file mode 100644 index bd76dc4..0000000 --- a/jdluc/__tests__/test_extract_huang_bgb.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Unit tests for extract/huang_bgb.py. - -Mocks HTTP + zip + NetCDF → GeoTIFF + GCS + ingestion so the tests run -offline. Guards the Design Decision 4 migration: no reproject / NoData -fill at extract (that moved to transform/emissions.py). -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import huang_bgb -from jdluc.utils.constants import GEE_HUANG_BGB, HUANG_BGB_FIGSHARE_URL - - -@pytest.fixture -def mock_io() -> Any: - """Patch every _io helper used by huang_bgb with MagicMocks.""" - with ( - patch.object(huang_bgb, 'delete_asset_if_present') as delete_asset, - patch.object(huang_bgb, 'delete_gcs_blob') as delete_gcs, - patch.object(huang_bgb, 'fetch_with_mirror') as fetch, - patch.object(huang_bgb, 'start_ingestion_and_wait') as ingest, - patch.object(huang_bgb, 'upload_to_gcs') as upload, - ): - - ns = MagicMock() - ns.delete_asset = delete_asset - ns.delete_gcs = delete_gcs - ns.fetch = fetch - ns.ingest = ingest - ns.upload = upload - upload.side_effect = lambda project, bucket, blob, path: f'gs://{bucket}/{blob}' - yield ns - - -def test_extract_huang_bgb_happy_path(mock_io: Any) -> None: - with ( - patch.object(huang_bgb, '_extract_netcdf') as extract_nc, - patch.object(huang_bgb, '_write_conus_geotiff') as write_tif, - ): - extract_nc.return_value = '/tmp/fake/pergridarea_bgb.nc' - asset_id = huang_bgb.extract_huang_bgb(gcp_project='ws-dev') - - assert asset_id == GEE_HUANG_BGB - mock_io.fetch.assert_called_once() - fetch_kwargs = mock_io.fetch.call_args.kwargs - assert fetch_kwargs['dataset'] == 'huang_bgb' - assert fetch_kwargs['filename'] == 'data_code_to_submit.zip' - assert fetch_kwargs['source'] == HUANG_BGB_FIGSHARE_URL - assert fetch_kwargs['gcp_project'] == 'ws-dev' - - extract_nc.assert_called_once() - write_tif.assert_called_once() - - mock_io.upload.assert_called_once() - mock_io.ingest.assert_called_once() - ingest_kwargs = mock_io.ingest.call_args.kwargs - assert mock_io.ingest.call_args.args[1] == GEE_HUANG_BGB - assert ingest_kwargs['band_name'] == huang_bgb.BAND_NAME - assert ingest_kwargs['allow_overwrite'] is False - - mock_io.delete_gcs.assert_called_once() - mock_io.delete_asset.assert_not_called() - - -def test_extract_huang_bgb_force_deletes_existing_asset_first(mock_io: Any) -> None: - with ( - patch.object(huang_bgb, '_extract_netcdf') as extract_nc, - patch.object(huang_bgb, '_write_conus_geotiff'), - ): - extract_nc.return_value = '/tmp/fake/pergridarea_bgb.nc' - huang_bgb.extract_huang_bgb(gcp_project='ws-dev', force=True) - - # force=True → delete existing asset before starting. - mock_io.delete_asset.assert_called_once_with(GEE_HUANG_BGB) - # And pass through to the ingest with allow_overwrite=True. - assert mock_io.ingest.call_args.kwargs['allow_overwrite'] is True - - -def test_extract_netcdf_missing_member_raises(tmp_path: Any) -> None: - import zipfile - - zip_path = tmp_path / 'test.zip' - with zipfile.ZipFile(zip_path, 'w') as zf: - zf.writestr('README.md', 'nothing to see here') - - with pytest.raises(RuntimeError, match='not found in archive'): - huang_bgb._extract_netcdf(str(zip_path), str(tmp_path)) - - -def test_extract_huang_bgb_does_not_reproject_or_gap_fill() -> None: - # CRS + NoData handling lives in transform/emissions.py, not extract. - # Guard the public surface so a future contributor doesn't silently - # reintroduce reprojection here. - import inspect - - src = inspect.getsource(huang_bgb) - assert '.reproject(' not in src - assert '.unmask(' not in src - - -def test_conus_bounds_are_mainland_usa() -> None: - assert huang_bgb.CONUS_WEST == -130.0 - assert huang_bgb.CONUS_EAST == -65.0 - assert huang_bgb.CONUS_SOUTH == 24.0 - assert huang_bgb.CONUS_NORTH == 50.0 diff --git a/jdluc/__tests__/test_extract_integration.py b/jdluc/__tests__/test_extract_integration.py deleted file mode 100644 index a62137b..0000000 --- a/jdluc/__tests__/test_extract_integration.py +++ /dev/null @@ -1,120 +0,0 @@ -"""End-to-end integration test for the extract stage. - -Marked ``@pytest.mark.integration`` — requires live GEE credentials and -write access to the GCS staging bucket. Only IPCC climate zones is -exercised end-to-end (it is the smallest of the four datasets, with -~18 KB source and sub-minute full round-trip). Harris / Huang / NASS -end-to-end runs are covered by the per-module unit tests with mocks -plus the BigQuery publish path and CONUS pipeline run as real-scale -validators. - -The test is DESTRUCTIVE against the GEE IPCC asset (deletes + rebuilds -it). Do not run against production if anything else depends on the -asset being continuously available. -""" - -from collections import Counter -from typing import Any - -import ee -import pytest - -from jdluc.extract.extract import extract_all -from jdluc.utils.constants import ( - GCP_PROJECT, - GEE_IPCC_CLIMATE_ZONES, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - NON_NATIVE_DATASETS, -) -from jdluc.utils.gee import ( - asset_exists, - delete_asset_if_present, - initialize_gee, -) - -# Delaware bounding box for the histogram snapshot. Kept small so the -# reducer runs in a handful of seconds. -_DELAWARE_BBOX: list[float] = [-75.8, 38.4, -74.9, 39.9] - - -def _ipcc_histogram_over_delaware() -> Counter[int]: - """Return a {climate_zone_code: pixel_count} histogram over Delaware. - - Uses the GLAD_CRS_TRANSFORM so the sampling grid matches what - transform/emissions.py actually reads. - """ - image = ee.Image(GEE_IPCC_CLIMATE_ZONES) - region = ee.Geometry.Rectangle(_DELAWARE_BBOX, proj=GLAD_CRS, geodesic=False) - # frequencyHistogram returns {stringified_value: count}. - result = image.reduceRegion( - reducer=ee.Reducer.frequencyHistogram(), - geometry=region, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e9), - bestEffort=True, - ).getInfo() - raw = next(iter(result.values())) if result else {} - return Counter({int(k): int(v) for k, v in (raw or {}).items()}) - - -@pytest.fixture(scope='module') -def _ee_initialized() -> None: - initialize_gee(GCP_PROJECT) - - -@pytest.mark.integration -def test_ipcc_end_to_end_rebuild_matches_snapshot(_ee_initialized: None) -> None: - """Delete + re-extract IPCC, confirm histogram matches pre-deletion.""" - if not asset_exists(GEE_IPCC_CLIMATE_ZONES): - pytest.skip( - f'No baseline {GEE_IPCC_CLIMATE_ZONES} asset to snapshot against. ' - f'Run the orchestrator once before the test to seed it.' - ) - - baseline = _ipcc_histogram_over_delaware() - assert sum(baseline.values()) > 0, 'baseline histogram is empty' - - delete_asset_if_present(GEE_IPCC_CLIMATE_ZONES) - - result = extract_all(gcp_project=GCP_PROJECT, force=False) - assert 'ipcc_climate_zones' in result.extracted, result - assert result.failed == {}, result.failed - assert asset_exists(GEE_IPCC_CLIMATE_ZONES) - - rebuilt = _ipcc_histogram_over_delaware() - # Exact match — same source, same reprojection, categorical nearest- - # neighbor resampling is deterministic. - assert rebuilt == baseline, (rebuilt, baseline) - - -@pytest.mark.integration -def test_extract_all_is_idempotent_on_second_run(_ee_initialized: None) -> None: - """Second force=False invocation finds every asset cached, does no work.""" - # Pre-condition: every expected asset should already exist from the - # previous test (or a prior pipeline run). - for family in NON_NATIVE_DATASETS: - from jdluc.utils.constants import DATASET_INVENTORY - - aid = DATASET_INVENTORY[family]['gee_asset_id'] - if not asset_exists(aid): - pytest.skip(f'precondition: asset missing for {family} ({aid})') - - result = extract_all(gcp_project=GCP_PROJECT, force=False) - assert sorted(result.cached) == sorted(NON_NATIVE_DATASETS) - assert result.extracted == [] - assert result.failed == {} - - -def test_extract_integration_test_module_imports_cleanly() -> None: - """Trivial non-integration sanity check — catches import-time regressions - in the integration test module so they surface in the fast CI pass - rather than only when the integration suite runs. - """ - # Guards the method under test exists on the public surface the - # integration body depends on. - assert callable(extract_all) - assert callable(_ipcc_histogram_over_delaware) - assert len(_DELAWARE_BBOX) == 4 - _: Any = ee # Silence "imported but unused" for the integration branch. diff --git a/jdluc/__tests__/test_extract_ipcc_climate_zones.py b/jdluc/__tests__/test_extract_ipcc_climate_zones.py deleted file mode 100644 index ebfc224..0000000 --- a/jdluc/__tests__/test_extract_ipcc_climate_zones.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Unit tests for extract/ipcc_climate_zones.py. - -Mocks Zenodo HTTP + GCS + GEE so the tests run offline. Covers the -scratch-then-reproject two-step upload flow and the cleanup contract. -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import ipcc_climate_zones -from jdluc.utils.constants import ( - GEE_IPCC_CLIMATE_ZONES, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - IPCC_CLIMATE_ZONES_ZENODO_URL, -) - - -@pytest.fixture -def mock_io() -> Any: - with ( - patch.object(ipcc_climate_zones, 'asset_exists') as asset_exists, - patch.object(ipcc_climate_zones, 'delete_asset_if_present') as delete_asset, - patch.object(ipcc_climate_zones, 'delete_gcs_blob') as delete_gcs, - patch.object(ipcc_climate_zones, 'fetch_with_mirror') as fetch, - patch.object(ipcc_climate_zones, 'start_ingestion_and_wait') as ingest, - patch.object(ipcc_climate_zones, 'upload_to_gcs') as upload, - patch.object(ipcc_climate_zones, 'wait_for_export_task') as wait_export, - ): - ns = MagicMock() - ns.asset_exists = asset_exists - ns.delete_asset = delete_asset - ns.delete_gcs = delete_gcs - ns.fetch = fetch - ns.ingest = ingest - ns.upload = upload - ns.wait_export = wait_export - asset_exists.return_value = False - upload.side_effect = lambda project, bucket, blob, path: f'gs://{bucket}/{blob}' - yield ns - - -def _fake_zenodo_record(tif_key: str = 'CLIMATE_ZONE.tif') -> Any: - resp = MagicMock() - resp.raise_for_status = MagicMock() - resp.json = MagicMock( - return_value={ - 'files': [ - { - 'key': 'README.md', - 'links': {'self': 'https://zenodo/readme'}, - }, - { - 'key': tif_key, - 'links': {'self': f'https://zenodo/{tif_key}'}, - }, - ] - } - ) - return resp - - -def test_discover_zenodo_tif_url_finds_single_tif() -> None: - with patch('jdluc.extract.ipcc_climate_zones.requests.get') as get: - get.return_value = _fake_zenodo_record() - filename, url = ipcc_climate_zones._discover_zenodo_tif_url() - - assert filename.lower().endswith('.tif') - assert url == 'https://zenodo/CLIMATE_ZONE.tif' - assert get.call_args.args[0] == IPCC_CLIMATE_ZONES_ZENODO_URL - - -def test_discover_zenodo_tif_url_rejects_multiple_tifs() -> None: - resp = MagicMock() - resp.raise_for_status = MagicMock() - resp.json = MagicMock( - return_value={ - 'files': [ - {'key': 'a.tif', 'links': {'self': 'x'}}, - {'key': 'b.tif', 'links': {'self': 'y'}}, - ] - } - ) - with ( - patch( - 'jdluc.extract.ipcc_climate_zones.requests.get', - return_value=resp, - ), - pytest.raises(RuntimeError, match='expected to contain exactly 1 .tif'), - ): - ipcc_climate_zones._discover_zenodo_tif_url() - - -def test_extract_ipcc_runs_ingest_then_reproject_then_cleanup(mock_io: Any) -> None: - export_task = MagicMock() - - with ( - patch.object( - ipcc_climate_zones, - '_discover_zenodo_tif_url', - return_value=('CLIMATE_ZONE.tif', 'https://zenodo/CLIMATE_ZONE.tif'), - ), - patch.object( - ipcc_climate_zones, - '_start_reproject_export', - return_value=export_task, - ) as start_export, - ): - asset_id = ipcc_climate_zones.extract_ipcc_climate_zones(gcp_project='ws-dev') - - assert asset_id == GEE_IPCC_CLIMATE_ZONES - mock_io.fetch.assert_called_once() - fetch_kwargs = mock_io.fetch.call_args.kwargs - assert fetch_kwargs['dataset'] == 'ipcc_climate_zones' - assert fetch_kwargs['filename'] == ipcc_climate_zones.SOURCE_FILENAME - # IPCC uses a deferred-discovery callable source. - assert callable(fetch_kwargs['source']) - mock_io.upload.assert_called_once() - mock_io.ingest.assert_called_once() - start_export.assert_called_once_with( - ipcc_climate_zones.SCRATCH_ASSET_ID, GEE_IPCC_CLIMATE_ZONES - ) - mock_io.wait_export.assert_called_once() - # GCS blob is cleaned up (in a try/finally around the ingest+export). - mock_io.delete_gcs.assert_called_once() - # Scratch asset is deleted after the final asset is written. - mock_io.delete_asset.assert_called_with(ipcc_climate_zones.SCRATCH_ASSET_ID) - - -def test_extract_ipcc_force_deletes_scratch_and_final_first(mock_io: Any) -> None: - with ( - patch.object( - ipcc_climate_zones, - '_discover_zenodo_tif_url', - return_value=('CLIMATE_ZONE.tif', 'https://zenodo/CLIMATE_ZONE.tif'), - ), - patch.object(ipcc_climate_zones, '_start_reproject_export') as start_export, - ): - start_export.return_value = MagicMock() - ipcc_climate_zones.extract_ipcc_climate_zones(gcp_project='ws-dev', force=True) - - # Both scratch and final assets are deleted at the top on force=True. - delete_targets = [call.args[0] for call in mock_io.delete_asset.call_args_list] - assert ipcc_climate_zones.SCRATCH_ASSET_ID in delete_targets - assert GEE_IPCC_CLIMATE_ZONES in delete_targets - # ingest is invoked with allow_overwrite=True on force. - assert mock_io.ingest.call_args.kwargs['allow_overwrite'] is True - - -def test_extract_ipcc_clears_stale_scratch_when_not_forcing(mock_io: Any) -> None: - # asset_exists returns True for the scratch probe only — simulates a - # prior partial run that left scratch but not the final asset. - mock_io.asset_exists.side_effect = lambda aid: ( - aid == ipcc_climate_zones.SCRATCH_ASSET_ID - ) - - with ( - patch.object( - ipcc_climate_zones, - '_discover_zenodo_tif_url', - return_value=('CLIMATE_ZONE.tif', 'https://zenodo/CLIMATE_ZONE.tif'), - ), - patch.object( - ipcc_climate_zones, '_start_reproject_export', return_value=MagicMock() - ), - ): - ipcc_climate_zones.extract_ipcc_climate_zones(gcp_project='ws-dev') - - # Expect an extra delete_asset_if_present(scratch) at the top. - scratch_deletes = [ - c - for c in mock_io.delete_asset.call_args_list - if c.args[0] == ipcc_climate_zones.SCRATCH_ASSET_ID - ] - assert len(scratch_deletes) >= 1 - - -def test_reproject_export_targets_glad_grid() -> None: - # Guards the methodology invariant: IPCC final asset is pinned to the - # GLAD 0.00025° grid. Mocks ee so the test runs without credentials. - with patch('jdluc.extract.ipcc_climate_zones.ee') as ee_mock: - image = MagicMock() - ee_mock.Image.return_value = image - ee_mock.Geometry.Rectangle.return_value = 'region' - export_task = MagicMock() - ee_mock.batch.Export.image.toAsset.return_value = export_task - image.reproject.return_value = 'reprojected' - - result = ipcc_climate_zones._start_reproject_export('src-asset', 'dst-asset') - - assert result is export_task - image.reproject.assert_called_once_with( - crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM - ) - kwargs = ee_mock.batch.Export.image.toAsset.call_args.kwargs - assert kwargs['assetId'] == 'dst-asset' - assert kwargs['crs'] == GLAD_CRS - assert kwargs['crsTransform'] == GLAD_CRS_TRANSFORM - export_task.start.assert_called_once() diff --git a/jdluc/__tests__/test_extract_mirror.py b/jdluc/__tests__/test_extract_mirror.py deleted file mode 100644 index 555f551..0000000 --- a/jdluc/__tests__/test_extract_mirror.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Unit tests for extract/mirror.py. - -Mocks the google.cloud.storage client and the underlying -download_with_retries / upload_to_gcs helpers so these tests run fully -offline. Covers: mirror-hit short-circuits upstream; mirror-miss falls -through to upstream + writes the mirror; callable source is only -resolved on miss; errors propagate sanely. -""" - -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.extract import mirror -from jdluc.utils.constants import GCS_BUCKET_NAME - - -def test_mirror_hit_short_circuits_upstream(tmp_path: object) -> None: - """On mirror hit, upstream is never called and the blob is streamed down.""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - def resolver() -> str: - raise AssertionError('resolver should not be called on mirror hit') - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=True) as exists_mock, - patch.object(mirror, '_download_from_gcs') as download_mock, - patch.object(mirror, 'download_with_retries') as upstream_mock, - patch.object(mirror, 'upload_to_gcs') as upload_mock, - ): - mirror.fetch_with_mirror( - dst, - dataset='nass_yields', - filename='qs.crops_20260423.txt.gz', - gcp_project='gcp-project-name', - source=resolver, - ) - - exists_mock.assert_called_once_with( - 'gcp-project-name', - GCS_BUCKET_NAME, - 'luc_high_res/extract_mirror/nass_yields/qs.crops_20260423.txt.gz', - ) - download_mock.assert_called_once_with( - 'gcp-project-name', - GCS_BUCKET_NAME, - 'luc_high_res/extract_mirror/nass_yields/qs.crops_20260423.txt.gz', - dst, - ) - upstream_mock.assert_not_called() - upload_mock.assert_not_called() - - -def test_mirror_miss_fetches_upstream_and_writes_mirror(tmp_path: object) -> None: - """On mirror miss, upstream is called and the result uploads to GCS.""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=False), - patch.object(mirror, '_download_from_gcs') as gcs_download_mock, - patch.object(mirror, 'download_with_retries') as upstream_mock, - patch.object(mirror, 'upload_to_gcs') as upload_mock, - ): - mirror.fetch_with_mirror( - dst, - dataset='huang_bgb', - filename='data_code_to_submit.zip', - gcp_project='gcp-project-name', - source='https://example/huang.zip', - timeout_s=900.0, - ) - - gcs_download_mock.assert_not_called() - upstream_mock.assert_called_once_with( - 'https://example/huang.zip', - dst, - timeout_s=900.0, - retries=3, - backoff_s=2.0, - ) - upload_mock.assert_called_once_with( - 'gcp-project-name', - GCS_BUCKET_NAME, - 'luc_high_res/extract_mirror/huang_bgb/data_code_to_submit.zip', - dst, - ) - - -def test_mirror_miss_resolves_callable_source(tmp_path: object) -> None: - """Deferred-resolution sources get invoked exactly once on miss.""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - resolver = MagicMock(return_value='https://zenodo/files/record.tif') - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=False), - patch.object(mirror, 'download_with_retries'), - patch.object(mirror, 'upload_to_gcs'), - ): - mirror.fetch_with_mirror( - dst, - dataset='ipcc_climate_zones', - filename='ipcc_climate_zones_v2006.tif', - gcp_project='gcp-project-name', - source=resolver, - ) - - resolver.assert_called_once_with() - - -def test_mirror_hit_does_not_resolve_callable_source(tmp_path: object) -> None: - """On hit, a callable source is never invoked (no Zenodo probe).""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - resolver = MagicMock(return_value='irrelevant') - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=True), - patch.object(mirror, '_download_from_gcs'), - ): - mirror.fetch_with_mirror( - dst, - dataset='ipcc_climate_zones', - filename='ipcc_climate_zones_v2006.tif', - gcp_project='gcp-project-name', - source=resolver, - ) - - resolver.assert_not_called() - - -def test_mirror_miss_upstream_failure_propagates(tmp_path: object) -> None: - """If upstream download raises on mirror miss, the error bubbles up.""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=False), - patch.object( - mirror, - 'download_with_retries', - side_effect=RuntimeError('upstream 404'), - ), - patch.object(mirror, 'upload_to_gcs') as upload_mock, - ): - with pytest.raises(RuntimeError, match='upstream 404'): - mirror.fetch_with_mirror( - dst, - dataset='nass_yields', - filename='qs.crops_missing.txt.gz', - gcp_project='gcp-project-name', - source='https://nass/missing', - ) - - # Upload must NOT run if the upstream download failed. - upload_mock.assert_not_called() - - -def test_mirror_retry_kwargs_pass_through(tmp_path: object) -> None: - """timeout_s / retries / backoff_s forward to download_with_retries.""" - dst = str(tmp_path / 'payload.bin') # type: ignore[operator] - - with ( - patch.object(mirror, '_gcs_blob_exists', return_value=False), - patch.object(mirror, 'download_with_retries') as upstream_mock, - patch.object(mirror, 'upload_to_gcs'), - ): - mirror.fetch_with_mirror( - dst, - dataset='harris_agb', - filename='40N_080W.tif', - gcp_project='gcp-project-name', - source='https://gfw/tile', - timeout_s=300.0, - retries=5, - backoff_s=1.5, - ) - - kwargs = upstream_mock.call_args.kwargs - assert kwargs['timeout_s'] == 300.0 - assert kwargs['retries'] == 5 - assert kwargs['backoff_s'] == 1.5 diff --git a/jdluc/__tests__/test_extract_nass_yields.py b/jdluc/__tests__/test_extract_nass_yields.py deleted file mode 100644 index b36ddc3..0000000 --- a/jdluc/__tests__/test_extract_nass_yields.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Unit tests for extract/nass_yields.py. - -Mocks HTTP + pandas chunk reader + EE/GCS so the tests run offline. -Covers the upload-size filter, the value-coercion rules, the rename -map that the transform consumer expects, and the GCS-staged -ingestion path that replaced the inline FeatureCollection upload -(which hit GEE's 10 MB request payload limit on full-archive runs). -""" - -import gzip -from typing import Any -from unittest.mock import patch - -import pandas as pd -import pytest - -from jdluc.extract import nass_yields -from jdluc.utils.constants import GCS_BUCKET_NAME, GEE_NASS_YIELDS - - -def _fake_tsv_frame() -> pd.DataFrame: - """Minimum-column chunk matching QuickStats' schema.""" - return pd.DataFrame( - [ - # YIELD-CORN-STATE — kept. - { - 'STATISTICCAT_DESC': 'YIELD', - 'COMMODITY_DESC': 'CORN', - 'AGG_LEVEL_DESC': 'STATE', - 'UNIT_DESC': 'BU / ACRE', - 'YEAR': '2020', - 'VALUE': '180.0', - 'STATE_FIPS_CODE': '10', - 'STATE_NAME': 'DELAWARE', - }, - # Unknown commodity — dropped by upload filter. - { - 'STATISTICCAT_DESC': 'YIELD', - 'COMMODITY_DESC': 'BARLEY', - 'AGG_LEVEL_DESC': 'STATE', - 'UNIT_DESC': 'BU / ACRE', - 'YEAR': '2020', - 'VALUE': '80.0', - 'STATE_FIPS_CODE': '10', - 'STATE_NAME': 'DELAWARE', - }, - # Non-YIELD STATISTICCAT — dropped. - { - 'STATISTICCAT_DESC': 'AREA PLANTED', - 'COMMODITY_DESC': 'CORN', - 'AGG_LEVEL_DESC': 'STATE', - 'UNIT_DESC': 'ACRES', - 'YEAR': '2020', - 'VALUE': '1000.0', - 'STATE_FIPS_CODE': '10', - 'STATE_NAME': 'DELAWARE', - }, - # Suppressed value token — dropped. - { - 'STATISTICCAT_DESC': 'YIELD', - 'COMMODITY_DESC': 'CORN', - 'AGG_LEVEL_DESC': 'STATE', - 'UNIT_DESC': 'BU / ACRE', - 'YEAR': '2020', - 'VALUE': '(D)', - 'STATE_FIPS_CODE': '10', - 'STATE_NAME': 'DELAWARE', - }, - # County-level row — kept at extract (transform filters it out). - { - 'STATISTICCAT_DESC': 'YIELD', - 'COMMODITY_DESC': 'SOYBEANS', - 'AGG_LEVEL_DESC': 'COUNTY', - 'UNIT_DESC': 'BU / ACRE', - 'YEAR': '2020', - 'VALUE': '48.5', - 'STATE_FIPS_CODE': '19', - 'STATE_NAME': 'IOWA', - }, - ] - ) - - -def test_download_and_parse_keeps_only_valid_commodity_yield_rows( - tmp_path: Any, -) -> None: - gzip_path = tmp_path / 'qs.fake.txt.gz' - # Write a trivial gzip blob so _download_and_parse's gzip.open works; - # we'll mock download_with_retries so no actual HTTP is fired, and - # mock pd.read_csv to return our synthetic chunk. - with gzip.open(gzip_path, 'wt') as fh: - fh.write('header\n') - - chunk = _fake_tsv_frame() - with ( - patch.object(nass_yields, 'fetch_with_mirror'), - patch( - 'jdluc.extract.nass_yields.pd.read_csv', - return_value=iter([chunk]), - ), - ): - df = nass_yields._download_and_parse(str(gzip_path), 'ws-dev') - - # CORN-STATE-YIELD, SOYBEANS-COUNTY-YIELD survive. BARLEY / - # AREA PLANTED / suppressed rows are dropped. - assert list(df.columns) == nass_yields.ROW_COLUMNS - commodities = sorted(df['commodity_desc'].unique().tolist()) - assert commodities == ['CORN', 'SOYBEANS'] - assert df['value_bu_per_acre'].dtype == float - assert df['year'].dtype == int - assert set(df['state_fips'].unique()) == {'10', '19'} - - -def test_download_and_parse_raises_when_nothing_matches(tmp_path: Any) -> None: - gzip_path = tmp_path / 'qs.empty.txt.gz' - with gzip.open(gzip_path, 'wt') as fh: - fh.write('header\n') - - empty_chunk = pd.DataFrame( - { - 'STATISTICCAT_DESC': ['AREA PLANTED'], - 'COMMODITY_DESC': ['CORN'], - 'AGG_LEVEL_DESC': ['STATE'], - 'UNIT_DESC': ['ACRES'], - 'YEAR': ['2020'], - 'VALUE': ['1'], - 'STATE_FIPS_CODE': ['10'], - 'STATE_NAME': ['DELAWARE'], - } - ) - with ( - patch.object(nass_yields, 'fetch_with_mirror'), - patch( - 'jdluc.extract.nass_yields.pd.read_csv', - return_value=iter([empty_chunk]), - ), - pytest.raises(RuntimeError, match='no YIELD rows'), - ): - nass_yields._download_and_parse(str(gzip_path), 'ws-dev') - - -def _sample_df() -> pd.DataFrame: - return pd.DataFrame( - [ - { - 'state_fips': '10', - 'state_name': 'DELAWARE', - 'commodity_desc': 'CORN', - 'year': 2020, - 'value_bu_per_acre': 180.0, - 'agg_level_desc': 'STATE', - 'unit_desc': 'BU / ACRE', - } - ] - ) - - -def test_extract_nass_yields_stages_csv_to_gcs_and_ingests() -> None: - df = _sample_df() - with ( - patch.object(nass_yields, '_download_and_parse', return_value=df), - patch.object( - nass_yields, - 'upload_to_gcs', - side_effect=lambda project, bucket, blob, path: f'gs://{bucket}/{blob}', - ) as upload, - patch.object(nass_yields, 'start_table_ingestion_and_wait') as ingest, - patch.object(nass_yields, 'delete_gcs_blob') as delete_gcs, - patch.object(nass_yields, 'delete_asset_if_present') as delete_asset, - ): - asset_id = nass_yields.extract_nass_yields(gcp_project='ws-dev') - - assert asset_id == GEE_NASS_YIELDS - delete_asset.assert_not_called() # force=False - - # CSV gets uploaded to a stable per-run GCS path under the staging - # prefix; the ingestion request reads from that GCS URI. - upload.assert_called_once() - project, bucket, blob, path = upload.call_args.args - assert project == 'ws-dev' - assert bucket == GCS_BUCKET_NAME - assert blob.startswith(f'{nass_yields.GCS_STAGING_PREFIX}/') - assert blob.endswith('/nass_yields_raw.csv') - # Local path written by pandas was a real CSV (deleted by cleanup - # before the assertion runs, so we can only check the upload arg). - assert path.endswith('.csv') - - ingest.assert_called_once() - args, kwargs = ingest.call_args.args, ingest.call_args.kwargs - assert args[0] == f'gs://{bucket}/{blob}' - assert args[1] == GEE_NASS_YIELDS - assert kwargs.get('allow_overwrite') is False - - # Staging blob is cleaned up after ingestion (success path). - delete_gcs.assert_called_once_with('ws-dev', bucket, blob) - - -def test_extract_nass_yields_force_clears_existing_asset() -> None: - df = _sample_df() - with ( - patch.object(nass_yields, '_download_and_parse', return_value=df), - patch.object( - nass_yields, - 'upload_to_gcs', - side_effect=lambda project, bucket, blob, path: f'gs://{bucket}/{blob}', - ), - patch.object(nass_yields, 'start_table_ingestion_and_wait') as ingest, - patch.object(nass_yields, 'delete_gcs_blob'), - patch.object(nass_yields, 'delete_asset_if_present') as delete_asset, - ): - nass_yields.extract_nass_yields(gcp_project='ws-dev', force=True) - - delete_asset.assert_called_once_with(GEE_NASS_YIELDS) - # force=True propagates to allow_overwrite on the ingestion request. - assert ingest.call_args.kwargs.get('allow_overwrite') is True - - -def test_extract_nass_yields_cleans_up_gcs_blob_on_ingestion_failure() -> None: - df = _sample_df() - with ( - patch.object(nass_yields, '_download_and_parse', return_value=df), - patch.object( - nass_yields, - 'upload_to_gcs', - side_effect=lambda project, bucket, blob, path: f'gs://{bucket}/{blob}', - ), - patch.object( - nass_yields, - 'start_table_ingestion_and_wait', - side_effect=RuntimeError('GEE 500'), - ), - patch.object(nass_yields, 'delete_gcs_blob') as delete_gcs, - patch.object(nass_yields, 'delete_asset_if_present'), - ): - with pytest.raises(RuntimeError, match='GEE 500'): - nass_yields.extract_nass_yields(gcp_project='ws-dev') - - # GCS blob still gets cleaned up on the failure path so we don't - # leak staging objects when the orchestrator records the failure - # and continues with other datasets. - delete_gcs.assert_called_once() - - -def test_row_columns_match_transform_consumer_schema() -> None: - # transform/summary_tables.py::_filter_average_convert_nass_rows reads - # exactly these keys — lock the schema so a later drift is caught here. - assert set(nass_yields.ROW_COLUMNS) == { - 'state_fips', - 'state_name', - 'commodity_desc', - 'year', - 'value_bu_per_acre', - 'agg_level_desc', - 'unit_desc', - } diff --git a/jdluc/__tests__/test_gcs.py b/jdluc/__tests__/test_gcs.py new file mode 100644 index 0000000..1d61bf1 --- /dev/null +++ b/jdluc/__tests__/test_gcs.py @@ -0,0 +1,16 @@ +import pytest + +from jdluc.gcs import ( + get_bucket_name_prefix_from_uri, + get_uri_from_bucket_name_prefix, +) + + +@pytest.mark.parametrize( + "bucket_name", ("bucket_name", "with-dash", "with space and .") +) +@pytest.mark.parametrize("prefix", ("file-at-root.txt", "path/to/nested/file.txt")) +def test_bucket_name_prefix_uri_roundtrip(bucket_name: str, prefix: str) -> None: + assert get_bucket_name_prefix_from_uri( + uri=get_uri_from_bucket_name_prefix(bucket_name=bucket_name, prefix=prefix) + ) == (bucket_name, prefix) diff --git a/jdluc/__tests__/test_geo.py b/jdluc/__tests__/test_geo.py new file mode 100644 index 0000000..b66a51b --- /dev/null +++ b/jdluc/__tests__/test_geo.py @@ -0,0 +1,64 @@ +import numpy +import pytest + +from jdluc.geo import ( + get_chunk_size, + get_overview_level, +) + + +@pytest.mark.parametrize( + ("height", "width", "minimum_pixels", "expected"), + ( + (1024, 1024, 1024, 0), + (2, 2, 1, 1), + (1024, 1024, 1, 10), + (1 << 32, 1 << 24, 1 << 6, 18), + ), +) +def test_get_overview_level( + height: int, width: int, minimum_pixels: int, expected: int +) -> None: + assert ( + get_overview_level(height=height, width=width, minimum_pixels=minimum_pixels) + == expected + ) + + +@pytest.mark.parametrize( + ("dtypes", "number_of_dimensions", "chunk_size"), + ( + ([numpy.dtype("uint8")], 1, 1 << 32), + ([numpy.dtype("uint8")], 2, 1 << 16), + ([numpy.dtype("uint8")], 3, 1 << 10), + ([numpy.dtype("uint8")], 4, 1 << 8), + ([numpy.dtype("uint16")], 1, 1 << 31), + ([numpy.dtype("uint16")], 2, 1 << 15), + ([numpy.dtype("uint16")], 3, 1 << 10), + ([numpy.dtype("uint16")], 4, 1 << 7), + ([numpy.dtype("float32")], 1, 1 << 30), + ([numpy.dtype("float32")], 2, 1 << 15), + ([numpy.dtype("float32")], 3, 1 << 10), + ([numpy.dtype("float32")], 4, 1 << 7), + ([numpy.dtype("float32")] * 1, 2, 1 << 15), + ([numpy.dtype("float32")] * 2, 2, 1 << 14), + ([numpy.dtype("float32")] * 3, 2, 1 << 14), + ([numpy.dtype("float32")] * 4, 2, 1 << 14), + ([numpy.dtype("float32")] * 5, 2, 1 << 13), + ([numpy.dtype("float32")] * 6, 2, 1 << 13), + ([numpy.dtype("float32")] * 7, 2, 1 << 13), + ([numpy.dtype("float32")] * 8, 2, 1 << 13), + ([numpy.dtype("float32")] * 9, 2, 1 << 13), + ([numpy.dtype("float32")] * 10, 2, 1 << 13), + ), +) +def test_get_chunk_size( + dtypes: list[numpy.dtype], number_of_dimensions: int, chunk_size: int +) -> None: + assert ( + get_chunk_size( + dtypes=dtypes, + number_of_dimensions=number_of_dimensions, + ) + == chunk_size + ) diff --git a/jdluc/__tests__/test_harmonize.py b/jdluc/__tests__/test_harmonize.py new file mode 100644 index 0000000..22d666d --- /dev/null +++ b/jdluc/__tests__/test_harmonize.py @@ -0,0 +1,113 @@ +import contextlib +import functools + +import pytest + +from jdluc.harmonize import GLAD_TILE_RESOLUTION, XY, Grid + + +@pytest.mark.parametrize( + ("x", "y", "fails"), + ( + (0, 0, False), + (100, 100, False), + (-1, -1, True), + ), +) +def test_xy_validated(x: int, y: int, fails: bool) -> None: + xy = XY(x=x, y=y) + context = ( + functools.partial(pytest.raises, AssertionError) + if fails + else contextlib.nullcontext + ) + with context(): + xy.validated() + + +@pytest.mark.parametrize( + ( + "tile_ids", + "origin", + "tiles", + "transform", + "resolution", + ), + ( + pytest.param( + ("00N_000W",), + XY(0, 0), + XY(1, 1), + (0, 1 / 4_000, 0, 0, 0, -1 / 4_000), + XY(40_000, 40_000), + id="One ten-degree-tile", + ), + pytest.param( + ("00N_000W", "00N_010E", "10S_000W", "10S_010E"), + XY(0, 0), + XY(2, 2), + (0, 1 / 4_000, 0, 0, 0, -1 / 4_000), + XY(2 * 40_000, 2 * 40_000), + id="2x2 ten-degree-tiles with NW=0,0", + ), + pytest.param( + ("60N_060W", "60S_060E"), + XY(-60, 60), + XY(13, 13), + (-60, 1 / 4_000, 0, 60, 0, -1 / 4_000), + XY(13 * 40_000, 13 * 40_000), + id="two tiles spanning a 13x13 area", + ), + ), +) +def test_grid_from_tile_ids_resolution( + tile_ids: tuple[str, ...], + origin: XY, + tiles: XY, + transform: tuple[float, float, float, float, float, float], + resolution: XY, +) -> None: + result = Grid.from_tile_ids_resolution( + tile_ids=tile_ids, tile_resolution=GLAD_TILE_RESOLUTION + ) + assert result.origin == origin + assert result.tiles == tiles + assert result.tile_resolution == GLAD_TILE_RESOLUTION + assert result.transform == transform + assert result.resolution == resolution + + +@pytest.mark.parametrize( + ("tile_id", "offset"), (("00N_000W", XY(0, 0)), ("90S_180E", XY(180, 90))) +) +def test_grid_get_offset_for_tile(tile_id: str, offset: XY) -> None: + grid = Grid.from_tile_ids_resolution( + tile_ids=("00N_000W",), tile_resolution=XY(10, 10) + ) + assert grid.get_offset_for_tile(tile_id=tile_id) == offset + + +def test_grid_get_offset_for_tile_raises() -> None: + grid = Grid.from_tile_ids_resolution( + tile_ids=("00N_000W",), tile_resolution=XY(10, 10) + ) + with pytest.raises(AssertionError): + grid.get_offset_for_tile(tile_id="10N_010W") + + +def test_grid_get_offset_for_world() -> None: + grid = Grid.from_tile_ids_resolution( + tile_ids=("60N_060W",), tile_resolution=XY(10, 10) + ) + assert grid.get_offset_for_world(resolution=XY(360, 180), span=XY(360, 180)) == XY( + 120, 30 + ) + + +def test_grid_get_resolution_for_world() -> None: + grid = Grid.from_tile_ids_resolution( + tile_ids=("60N_060W",), tile_resolution=XY(10, 10) + ) + assert grid.get_resolution_for_world( + resolution=XY(360, 180), span=XY(360, 180) + ) == XY(10, 10) diff --git a/jdluc/__tests__/test_pipeline.py b/jdluc/__tests__/test_pipeline.py deleted file mode 100644 index 2098745..0000000 --- a/jdluc/__tests__/test_pipeline.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Unit tests for pipeline.run_pipeline. - -Mocks extract_all, run_transform, run_publish, initialize_gee so the -tests run without GEE / BQ credentials. Covers the happy-path -propagation across all three stages, the fail-fast contracts on extract -and publish failures, and the from_cache aggregation rule. -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc import pipeline -from jdluc.extract.extract import ExtractError, ExtractResult -from jdluc.publish.publish import ( - BigQueryExportResult, - GCSExportResult, - PublishError, - PublishResult, -) -from jdluc.utils.constants import ( - BQ_CROPS_TABLE_PREFIX, - BQ_DATASET, - BQ_PROJECT, - BQ_TRANSITIONS_TABLE_PREFIX, -) - -_TRANSITIONS_TABLE_ID = ( - f'{BQ_PROJECT}.{BQ_DATASET}.{BQ_TRANSITIONS_TABLE_PREFIX}_delaware_abc_def' -) -_CROPS_TABLE_ID = f'{BQ_PROJECT}.{BQ_DATASET}.{BQ_CROPS_TABLE_PREFIX}_delaware_abc_def' - - -def _fake_transform_result(**overrides: Any) -> MagicMock: - tr = MagicMock() - tr.version = 'abc123456789' - tr.land_use_asset_id = 'projects/.../land_use_delaware_abc' - tr.emissions_asset_id = 'projects/.../emissions_delaware_abc' - tr.transitions_table_id = 'projects/.../transitions_delaware_abc' - tr.crops_table_id = 'projects/.../crops_delaware_abc' - tr.from_cache = False - for key, value in overrides.items(): - setattr(tr, key, value) - return tr - - -def _fake_publish_result( - transitions_from_cache: bool = False, - crops_from_cache: bool = False, - transitions_error: str | None = None, - crops_error: str | None = None, -) -> PublishResult: - return PublishResult( - transitions=BigQueryExportResult( - table_id=_TRANSITIONS_TABLE_ID, - transform_version='abc', - publish_version='def', - from_cache=transitions_from_cache, - error=transitions_error, - ), - crops=BigQueryExportResult( - table_id=_CROPS_TABLE_ID, - transform_version='abc', - publish_version='def', - from_cache=crops_from_cache, - error=crops_error, - ), - land_use=GCSExportResult( - gcs_uri="...", - transform_version="...", - publish_version="...", - from_cache=True, - error=None, - ), - emissions=GCSExportResult( - gcs_uri="...", - transform_version="...", - publish_version="...", - from_cache=True, - error=None, - ), - transitions_csv=GCSExportResult( - gcs_uri="...", - transform_version="...", - publish_version="...", - from_cache=True, - error=None, - ), - crops_csv=GCSExportResult( - gcs_uri="...", - transform_version="...", - publish_version="...", - from_cache=True, - error=None, - ), - ) - - -def test_run_pipeline_propagates_all_three_stages() -> None: - extract_result = ExtractResult( - cached=['harris_agb', 'huang_bgb'], - extracted=['ipcc_climate_zones', 'nass_yields'], - failed={}, - ) - publish_result = _fake_publish_result() - - with ( - patch.object(pipeline, 'initialize_gee') as init_gee, - patch.object( - pipeline, 'extract_all', return_value=extract_result - ) as extract_mock, - patch.object( - pipeline, 'run_transform', return_value=_fake_transform_result() - ) as transform_mock, - patch.object( - pipeline, 'run_publish', return_value=publish_result - ) as publish_mock, - ): - result = pipeline.run_pipeline( - gcp_project='ws-dev', states=['10'], region_name='delaware' - ) - - init_gee.assert_called_once_with('ws-dev') - extract_mock.assert_called_once_with(gcp_project='ws-dev', force=False) - transform_mock.assert_called_once() - publish_mock.assert_called_once() - assert result.extract_result is extract_result - assert result.publish_result is publish_result - assert result.publish_result.transitions.table_id == _TRANSITIONS_TABLE_ID - assert result.publish_result.crops.table_id == _CROPS_TABLE_ID - assert result.land_use_asset_id == 'projects/.../land_use_delaware_abc' - - -def test_run_pipeline_force_propagates_through_all_stages() -> None: - with ( - patch.object(pipeline, 'initialize_gee'), - patch.object( - pipeline, 'extract_all', return_value=ExtractResult(cached=['a']) - ) as extract_mock, - patch.object( - pipeline, 'run_transform', return_value=_fake_transform_result() - ) as transform_mock, - patch.object( - pipeline, 'run_publish', return_value=_fake_publish_result() - ) as publish_mock, - ): - pipeline.run_pipeline( - gcp_project='ws-dev', - states=['10'], - region_name='delaware', - force=True, - ) - - assert extract_mock.call_args.kwargs['force'] is True - assert transform_mock.call_args.kwargs['force'] is True - assert publish_mock.call_args.kwargs['force'] is True - - -def test_run_pipeline_raises_extract_error_without_running_transform_or_publish() -> ( - None -): - failing_result = ExtractResult( - cached=['harris_agb'], - extracted=[], - failed={'nass_yields': 'HTTP 500'}, - ) - with ( - patch.object(pipeline, 'initialize_gee'), - patch.object(pipeline, 'extract_all', return_value=failing_result), - patch.object(pipeline, 'run_transform') as transform_mock, - patch.object(pipeline, 'run_publish') as publish_mock, - ): - with pytest.raises(ExtractError, match='nass_yields'): - pipeline.run_pipeline( - gcp_project='ws-dev', states=['10'], region_name='delaware' - ) - transform_mock.assert_not_called() - publish_mock.assert_not_called() - - -def test_run_pipeline_propagates_publish_error_with_transform_completed() -> None: - failing_publish = _fake_publish_result(transitions_error='BQ 403') - with ( - patch.object(pipeline, 'initialize_gee'), - patch.object(pipeline, 'extract_all', return_value=ExtractResult(cached=['a'])), - patch.object(pipeline, 'run_transform', return_value=_fake_transform_result()), - patch.object( - pipeline, 'run_publish', side_effect=PublishError(failing_publish) - ), - pytest.raises(PublishError, match='transitions'), - ): - pipeline.run_pipeline( - gcp_project='ws-dev', states=['10'], region_name='delaware' - ) - - -def test_from_cache_is_true_only_when_every_stage_was_cached() -> None: - cases = [ - # (transform_from_cache, transitions_from_cache, crops_from_cache, expected) - (True, True, True, True), - (False, True, True, False), - (True, False, True, False), - (True, True, False, False), - (False, False, False, False), - ] - for tf, ttc, tcc, expected in cases: - with ( - patch.object(pipeline, 'initialize_gee'), - patch.object(pipeline, 'extract_all', return_value=ExtractResult()), - patch.object( - pipeline, - 'run_transform', - return_value=_fake_transform_result(from_cache=tf), - ), - patch.object( - pipeline, - 'run_publish', - return_value=_fake_publish_result( - transitions_from_cache=ttc, crops_from_cache=tcc - ), - ), - ): - result = pipeline.run_pipeline( - gcp_project='ws-dev', states=['10'], region_name='delaware' - ) - assert result.from_cache is expected, (tf, ttc, tcc, expected) - - -def test_run_pipeline_aborts_if_transform_did_not_produce_tables() -> None: - # If run_transform somehow returns without populating - # transitions_table_id or crops_table_id, we should raise rather - # than feed None into run_publish. - incomplete = _fake_transform_result(transitions_table_id=None) - with ( - patch.object(pipeline, 'initialize_gee'), - patch.object(pipeline, 'extract_all', return_value=ExtractResult()), - patch.object(pipeline, 'run_transform', return_value=incomplete), - patch.object(pipeline, 'run_publish') as publish_mock, - ): - with pytest.raises(RuntimeError, match='publish stage cannot proceed'): - pipeline.run_pipeline( - gcp_project='ws-dev', states=['10'], region_name='delaware' - ) - publish_mock.assert_not_called() diff --git a/jdluc/__tests__/test_publish_bigquery.py b/jdluc/__tests__/test_publish_bigquery.py deleted file mode 100644 index a926c17..0000000 --- a/jdluc/__tests__/test_publish_bigquery.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Unit tests for ``publish/bigquery.py``. - -Mocks ``google-cloud-bigquery`` + ``ee.batch.Export.table.toBigQuery`` -so every path runs offline. Covers table-ID format invariance, -cache-hit skip, force bypass, and the per-table wrappers funneling -through the shared primitive. -""" - -from collections.abc import Callable -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.publish import bigquery as pub_bq -from jdluc.publish.publish import BigQueryExportResult -from jdluc.utils.constants import ( - BQ_CROPS_TABLE_PREFIX, - BQ_DATASET, - BQ_PROJECT, - BQ_TRANSITIONS_TABLE_PREFIX, -) - -_T_SHA = 'a0d76ac0aa12' -_P_SHA = 'f91c22ab77d8' -_DIRTY_T_SHA = 'a0d76ac0aa12-dirty-3f2b8c01' - - -# ---------- build_*_bq_table_id -------------------------------------------- - - -def test_build_transitions_bq_table_id_format() -> None: - table_id = pub_bq.build_transitions_bq_table_id('delaware', _T_SHA, _P_SHA) - assert ( - table_id == f'{BQ_PROJECT}.{BQ_DATASET}.{BQ_TRANSITIONS_TABLE_PREFIX}' - f'_delaware_{_T_SHA}_{_P_SHA}' - ) - - -def test_build_crops_bq_table_id_format() -> None: - table_id = pub_bq.build_crops_bq_table_id('iowa', _T_SHA, _P_SHA) - assert ( - table_id == f'{BQ_PROJECT}.{BQ_DATASET}.{BQ_CROPS_TABLE_PREFIX}' - f'_iowa_{_T_SHA}_{_P_SHA}' - ) - - -def test_table_ids_survive_dirty_sha_components() -> None: - # The compound parser has to handle a dirty transform SHA + clean - # publish SHA at export time; table-id builder must produce a name - # that parse_compound_version_from_asset_id can round-trip. - from jdluc.utils.asset_management import ( - parse_compound_version_from_asset_id, - ) - - table_id = pub_bq.build_transitions_bq_table_id('delaware', _DIRTY_T_SHA, _P_SHA) - parsed = parse_compound_version_from_asset_id(table_id) - assert parsed == (_DIRTY_T_SHA, _P_SHA) - - -def test_prefixes_dont_overlap() -> None: - transitions = pub_bq.build_transitions_bq_table_id('delaware', _T_SHA, _P_SHA) - crops = pub_bq.build_crops_bq_table_id('delaware', _T_SHA, _P_SHA) - assert transitions != crops - # Same (region, t, p) triplet across both prefixes — only the prefix differs. - assert ( - transitions.replace(BQ_TRANSITIONS_TABLE_PREFIX, BQ_CROPS_TABLE_PREFIX) == crops - ) - - -# ---------- bq_table_exists ------------------------------------------------ - - -def _patch_bq_client() -> Any: - return patch('google.cloud.bigquery.Client') - - -def test_bq_table_exists_true_when_get_table_succeeds() -> None: - with _patch_bq_client() as mock_client: - mock_client.return_value.get_table = MagicMock(return_value=MagicMock()) - assert pub_bq.bq_table_exists('p.d.t') is True - - -def test_bq_table_exists_false_on_notfound() -> None: - from google.api_core import exceptions as gcs_exceptions - - with _patch_bq_client() as mock_client: - mock_client.return_value.get_table.side_effect = gcs_exceptions.NotFound( - 'no such table' - ) - assert pub_bq.bq_table_exists('p.d.t') is False - - -# ---------- ensure_bq_dataset_exists --------------------------------------- - - -def test_ensure_bq_dataset_exists_noop_when_dataset_present() -> None: - with _patch_bq_client() as mock_client: - mock_client.return_value.get_dataset = MagicMock(return_value=MagicMock()) - pub_bq.ensure_bq_dataset_exists() - mock_client.return_value.create_dataset.assert_not_called() - - -def test_ensure_bq_dataset_exists_creates_when_absent() -> None: - from google.api_core import exceptions as gcs_exceptions - - with _patch_bq_client() as mock_client: - mock_client.return_value.get_dataset.side_effect = gcs_exceptions.NotFound( - 'no such dataset' - ) - pub_bq.ensure_bq_dataset_exists() - mock_client.return_value.create_dataset.assert_called_once() - - -def test_ensure_bq_dataset_exists_swallows_create_race_conflict() -> None: - from google.api_core import exceptions as gcs_exceptions - - with _patch_bq_client() as mock_client: - mock_client.return_value.get_dataset.side_effect = gcs_exceptions.NotFound( - 'no such dataset' - ) - mock_client.return_value.create_dataset.side_effect = gcs_exceptions.Conflict( - 'someone else already created it' - ) - # Should not raise — race with parallel run is treated as success. - pub_bq.ensure_bq_dataset_exists() - - -# ---------- export_feature_collection_to_bigquery -------------------------- - - -def _valid_table_id(prefix: str = BQ_TRANSITIONS_TABLE_PREFIX) -> str: - return f'{BQ_PROJECT}.{BQ_DATASET}.{prefix}_delaware_{_T_SHA}_{_P_SHA}' - - -def test_export_primitive_rejects_table_id_without_compound_tail() -> None: - with pytest.raises(ValueError, match='compound'): - pub_bq.export_feature_collection_to_bigquery( - 'projects/.../transitions_delaware_abc', - 'not-a-compound-id', - ) - - -def test_export_primitive_cache_hit_skips_export() -> None: - table_id = _valid_table_id() - with ( - patch.object(pub_bq, 'bq_table_exists', return_value=True) as exists, - patch.object(pub_bq, 'ensure_bq_dataset_exists') as ensure_dataset, - patch('jdluc.publish.bigquery.ee.batch.Export.table.toBigQuery') as export, - patch.object(pub_bq, 'wait_for_export_task') as wait, - ): - result = pub_bq.export_feature_collection_to_bigquery( - 'projects/.../transitions_delaware_abc', - table_id, - force=False, - ) - - exists.assert_called_once_with(table_id) - # Cache hit returns before dataset existence check — avoids a needless - # round-trip on the warm path. - ensure_dataset.assert_not_called() - export.assert_not_called() - wait.assert_not_called() - assert result.from_cache is True - assert result.transform_version == _T_SHA - assert result.publish_version == _P_SHA - assert result.error is None - - -def test_export_primitive_force_bypasses_cache() -> None: - table_id = _valid_table_id() - fake_task = MagicMock() - with ( - patch.object(pub_bq, 'bq_table_exists', return_value=True), - patch.object(pub_bq, 'ensure_bq_dataset_exists') as ensure_dataset, - patch( - 'jdluc.publish.bigquery.ee.batch.Export.table.toBigQuery', - return_value=fake_task, - ) as export, - patch.object(pub_bq, 'wait_for_export_task') as wait, - patch( - 'jdluc.publish.bigquery.ee.FeatureCollection', - return_value='fc', - ), - ): - result = pub_bq.export_feature_collection_to_bigquery( - 'projects/.../transitions_delaware_abc', - table_id, - force=True, - ) - - ensure_dataset.assert_called_once() - export.assert_called_once() - export_kwargs = export.call_args.kwargs - assert export_kwargs['table'] == table_id - assert export_kwargs['overwrite'] is True - assert export_kwargs['append'] is False - fake_task.start.assert_called_once() - wait.assert_called_once() - assert result.from_cache is False - assert result.error is None - - -def test_export_primitive_cache_miss_runs_export() -> None: - table_id = _valid_table_id() - fake_task = MagicMock() - with ( - patch.object(pub_bq, 'bq_table_exists', return_value=False), - patch.object(pub_bq, 'ensure_bq_dataset_exists') as ensure_dataset, - patch( - 'jdluc.publish.bigquery.ee.batch.Export.table.toBigQuery', - return_value=fake_task, - ), - patch.object(pub_bq, 'wait_for_export_task') as wait, - patch( - 'jdluc.publish.bigquery.ee.FeatureCollection', - return_value='fc', - ), - ): - result = pub_bq.export_feature_collection_to_bigquery( - 'projects/.../transitions_delaware_abc', - table_id, - force=False, - ) - - ensure_dataset.assert_called_once() - fake_task.start.assert_called_once() - wait.assert_called_once() - assert result.from_cache is False - assert result.error is None - - -def test_export_primitive_captures_task_failure_as_error_field() -> None: - table_id = _valid_table_id() - fake_task = MagicMock() - with ( - patch.object(pub_bq, 'bq_table_exists', return_value=False), - patch.object(pub_bq, 'ensure_bq_dataset_exists'), - patch( - 'jdluc.publish.bigquery.ee.batch.Export.table.toBigQuery', - return_value=fake_task, - ), - patch.object(pub_bq, 'wait_for_export_task', side_effect=RuntimeError('boom')), - patch( - 'jdluc.publish.bigquery.ee.FeatureCollection', - return_value='fc', - ), - ): - result = pub_bq.export_feature_collection_to_bigquery( - 'projects/.../transitions_delaware_abc', - table_id, - force=False, - ) - - assert result.from_cache is False - assert result.error == 'boom' - assert result.transform_version == _T_SHA - assert result.publish_version == _P_SHA - - -# ---------- Per-table wrappers funnel through the primitive ---------------- - - -@pytest.mark.parametrize( - 'wrapper, prefix', - [ - (pub_bq.export_transitions_to_bigquery, BQ_TRANSITIONS_TABLE_PREFIX), - (pub_bq.export_crops_to_bigquery, BQ_CROPS_TABLE_PREFIX), - ], -) -def test_per_table_wrappers_delegate_to_shared_primitive( - wrapper: Callable[..., BigQueryExportResult], prefix: str -) -> None: - table_id = _valid_table_id(prefix=prefix) - expected = BigQueryExportResult( - table_id=table_id, - transform_version=_T_SHA, - publish_version=_P_SHA, - from_cache=True, - ) - with patch.object( - pub_bq, 'export_feature_collection_to_bigquery', return_value=expected - ) as primitive: - result = wrapper('projects/.../asset', table_id, force=True) - - primitive.assert_called_once_with('projects/.../asset', table_id, force=True) - assert result is expected diff --git a/jdluc/__tests__/test_publish_constants_and_shapes.py b/jdluc/__tests__/test_publish_constants_and_shapes.py deleted file mode 100644 index 390f677..0000000 --- a/jdluc/__tests__/test_publish_constants_and_shapes.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Unit tests for publish-stage constants and dataclass shapes. - -Covers (a) the BQ destination constants in ``utils/constants.py``, -(b) the compound-version parser extension in -``utils/asset_management.py``, and (c) the ``PublishResult`` / -``BigQueryExportResult`` / ``PublishError`` dataclass shapes. -""" - -import pytest - -from jdluc.utils import constants -from jdluc.utils.asset_management import ( - parse_compound_version_from_asset_id, - parse_version_from_asset_id, -) - -# ---------- BQ destination constants ---------------------------------------- - - -def test_bq_destination_constants_present() -> None: - # ``BQ_PROJECT`` / ``BQ_DATASET`` / ``BQ_JOB_LOCATION`` are deployment- - # configurable (edited directly in ``utils.constants``), so this test - # asserts shape — non-empty strings — rather than the Watershed defaults. - # The fixed table prefixes are still asserted exactly. - assert isinstance(constants.BQ_PROJECT, str) and constants.BQ_PROJECT - assert isinstance(constants.BQ_DATASET, str) and constants.BQ_DATASET - assert isinstance(constants.BQ_JOB_LOCATION, str) and constants.BQ_JOB_LOCATION - assert constants.BQ_TRANSITIONS_TABLE_PREFIX == 'luc_transitions' - assert constants.BQ_CROPS_TABLE_PREFIX == 'luc_crops' - - -def test_bq_prefixes_dont_collide_with_legacy() -> None: - # Legacy prefix was ``luc_emissions_summary``; the new prefixes must - # not share a prefix-string namespace so old and new tables coexist. - legacy = 'luc_emissions_summary' - assert not constants.BQ_TRANSITIONS_TABLE_PREFIX.startswith(legacy) - assert not constants.BQ_CROPS_TABLE_PREFIX.startswith(legacy) - assert not legacy.startswith(constants.BQ_TRANSITIONS_TABLE_PREFIX) - assert not legacy.startswith(constants.BQ_CROPS_TABLE_PREFIX) - - -# ---------- parse_compound_version_from_asset_id ---------------------------- - - -_T_SHA = 'a0d76ac0aa12' -_P_SHA = 'f91c22ab77d8' -_DIRTY_SHA = 'a0d76ac0aa12-dirty-3f2b8c01' - - -@pytest.mark.parametrize( - 'asset_id, expected', - [ - # Clean compound SHA pair — the canonical publish-asset shape. - ( - f'{constants.BQ_TRANSITIONS_TABLE_PREFIX}_delaware_{_T_SHA}_{_P_SHA}', - (_T_SHA, _P_SHA), - ), - # Dirty transform SHA, clean publish SHA. - ( - f'{constants.BQ_TRANSITIONS_TABLE_PREFIX}_delaware_{_DIRTY_SHA}_{_P_SHA}', - (_DIRTY_SHA, _P_SHA), - ), - # Clean transform SHA, dirty publish SHA. - ( - f'{constants.BQ_TRANSITIONS_TABLE_PREFIX}_delaware_{_T_SHA}_{_DIRTY_SHA}', - (_T_SHA, _DIRTY_SHA), - ), - # Full GEE-asset style path — the terminal segment is what matters. - ( - f'{constants.GEE_ASSET_ROOT}/' f'luc_crops_iowa_{_T_SHA}_{_P_SHA}', - (_T_SHA, _P_SHA), - ), - ], -) -def test_parse_compound_version_round_trips( - asset_id: str, expected: tuple[str, str] -) -> None: - assert parse_compound_version_from_asset_id(asset_id) == expected - - -def test_parse_compound_version_rejects_single_sha_assets() -> None: - # A transform-stage asset (one trailing SHA) must NOT be mis-parsed - # as a compound publish-asset. - single = f'{constants.GEE_ASSET_ROOT}/land_use_delaware_{_T_SHA}' - assert parse_compound_version_from_asset_id(single) is None - # Single-SHA parser still works on it. - assert parse_version_from_asset_id(single) == _T_SHA - - -def test_parse_compound_version_rejects_non_sha_trailing_segments() -> None: - assert ( - parse_compound_version_from_asset_id('luc_transitions_delaware_notasha') is None - ) - # Extract-style v-versions are not a compound either. - assert parse_compound_version_from_asset_id('harris_agb_conus_v2021') is None diff --git a/jdluc/__tests__/test_publish_integration.py b/jdluc/__tests__/test_publish_integration.py deleted file mode 100644 index e163fe6..0000000 --- a/jdluc/__tests__/test_publish_integration.py +++ /dev/null @@ -1,188 +0,0 @@ -"""End-to-end integration tests for the publish stage. - -Marked ``@pytest.mark.integration`` — requires live GEE credentials, -write access to the BigQuery destination dataset (``jdluc``), -and a previously-completed transform-stage run for Delaware (so the -transitions / crops GEE table assets exist for the export to read). - -Validates the four invariants the plan calls out: -1. End-to-end Delaware run lands both BQ tables with the right - schemas and a coherent (transform_sha, publish_sha) pair. -2. Cache-hit re-run: second invocation reports both targets cached - and queues no new BQ export tasks. -3. Publish-version bump produces a new pair of tables, leaves the - original pair intact. - -The legacy `test_aggregation_integration.py` / `test_export_integration.py` / -`test_bigquery_integration.py` covered the legacy aggregation / export / -bigquery modules; their behavior is now covered by -test_transform_*_integration.py (raster invariants), test_publish.py -(target dispatch), and the integration tests in this file. -""" - -from typing import Any -from unittest.mock import patch - -import pytest - -from jdluc import pipeline -from jdluc.publish.bigquery import bq_table_exists -from jdluc.publish.publish import run_publish -from jdluc.utils.asset_management import ( - parse_compound_version_from_asset_id, -) -from jdluc.utils.constants import ( - BQ_CROPS_TABLE_PREFIX, - BQ_TRANSITIONS_TABLE_PREFIX, - CROPS_TABLE_COLUMNS, - GCP_PROJECT, - GEE_ASSET_ROOT, - TRANSITIONS_TABLE_COLUMNS, -) -from jdluc.utils.gee import initialize_gee - -# Columns that GEE's Export.table.toBigQuery always materializes alongside -# the FeatureCollection's actual properties: `geo` is the WKB-encoded -# geometry (a Point(0, 0) placeholder for tabular outputs); `system:index` -# is the per-feature ID from the source FC. Neither is meaningful for the -# transitions / crops tables, but they're unavoidable in the BQ schema. -_GEE_BQ_AUTO_COLUMNS: set[str] = {'geo', 'system:index'} - - -def _bq_table_columns(table_id: str) -> set[str]: - """Return the column names of a BQ table, in any order.""" - from google.cloud import bigquery - - client = bigquery.Client(project=GCP_PROJECT) - table = client.get_table(table_id) - return {field.name for field in table.schema} - - -@pytest.fixture(scope='module') -def _ee_initialized() -> None: - initialize_gee(GCP_PROJECT) - - -@pytest.mark.integration -def test_end_to_end_delaware_publishes_both_bq_tables( - _ee_initialized: None, -) -> None: - """run_pipeline lands two BQ tables, both with correct schemas, and - both names parse back to the same (transform_sha, publish_sha) pair. - """ - result = pipeline.run_pipeline( - gcp_project=GCP_PROJECT, - states=['10'], - region_name='delaware', - force=False, - ) - - assert result.publish_result is not None - transitions_table = result.publish_result.transitions.table_id - crops_table = result.publish_result.crops.table_id - assert transitions_table is not None - assert crops_table is not None - assert bq_table_exists(transitions_table) - assert bq_table_exists(crops_table) - - # Schemas carry the canonical transitions / crops columns. BQ also - # auto-adds `geo` (the FeatureCollection geometry — a constant - # Point(0, 0) placeholder) and `system:index` (the GEE feature ID) - # when Export.table.toBigQuery materializes a FeatureCollection; - # both are inert for analysis but unavoidable. - transitions_cols = _bq_table_columns(transitions_table) - assert set(TRANSITIONS_TABLE_COLUMNS).issubset(transitions_cols), ( - set(TRANSITIONS_TABLE_COLUMNS) - transitions_cols - ) - assert _GEE_BQ_AUTO_COLUMNS.issubset(transitions_cols), transitions_cols - crops_cols = _bq_table_columns(crops_table) - assert set(CROPS_TABLE_COLUMNS).issubset(crops_cols), ( - set(CROPS_TABLE_COLUMNS) - crops_cols - ) - assert _GEE_BQ_AUTO_COLUMNS.issubset(crops_cols), crops_cols - - # Both tables share the same (transform_sha, publish_sha) pair — - # cross-table joins are safe. - transitions_versions = parse_compound_version_from_asset_id(transitions_table) - crops_versions = parse_compound_version_from_asset_id(crops_table) - assert transitions_versions is not None - assert transitions_versions == crops_versions - - # Sanity: transitions name carries the transitions prefix and crops - # name carries the crops prefix. - assert BQ_TRANSITIONS_TABLE_PREFIX in transitions_table - assert BQ_CROPS_TABLE_PREFIX in crops_table - - -@pytest.mark.integration -def test_pipeline_re_run_is_fully_cached(_ee_initialized: None) -> None: - """Second consecutive run with no code change reports from_cache=True - at every stage and queues no new BQ exports.""" - result = pipeline.run_pipeline( - gcp_project=GCP_PROJECT, - states=['10'], - region_name='delaware', - force=False, - ) - assert result.from_cache is True - assert result.publish_result is not None - assert result.publish_result.transitions.from_cache is True - assert result.publish_result.crops.from_cache is True - - -@pytest.mark.integration -def test_publish_version_bump_produces_new_tables(_ee_initialized: None) -> None: - """Monkey-patch compute_publish_version() to a synthetic SHA and - invoke run_publish directly. New tables must materialize at the new - name; the previous tables must still be intact.""" - from jdluc.publish import publish as publish_module - from jdluc.utils.version import compute_transform_version - - transform_version = compute_transform_version() - region = 'delaware' - transitions_asset = f'{GEE_ASSET_ROOT}/transitions_{region}_{transform_version}' - crops_asset = f'{GEE_ASSET_ROOT}/crops_{region}_{transform_version}' - - # Capture the "current code" run's table IDs first. - baseline = run_publish( - region_name=region, - transitions_asset_id=transitions_asset, - crops_asset_id=crops_asset, - land_use_asset_id="...", - emissions_asset_id="...", - transform_version=transform_version, - force=False, - ) - - bumped_publish_sha = 'deadbeef0000' - with patch.object( - publish_module, 'compute_publish_version', return_value=bumped_publish_sha - ): - bumped = run_publish( - region_name=region, - transitions_asset_id=transitions_asset, - crops_asset_id=crops_asset, - transform_version=transform_version, - land_use_asset_id="...", - emissions_asset_id="...", - force=False, - ) - - # Distinct table IDs (synthetic publish SHA in the suffix). - assert bumped.transitions.table_id != baseline.transitions.table_id - assert bumped.crops.table_id != baseline.crops.table_id - assert bumped.transitions.publish_version == bumped_publish_sha - assert bumped.crops.publish_version == bumped_publish_sha - # Both new tables exist, original tables still exist. - assert bq_table_exists(bumped.transitions.table_id) - assert bq_table_exists(bumped.crops.table_id) - assert bq_table_exists(baseline.transitions.table_id) - assert bq_table_exists(baseline.crops.table_id) - - -def test_publish_integration_module_imports_cleanly() -> None: - """Trivial non-integration sanity check — keeps import-time - regressions in this module surfacing in the fast CI pass.""" - assert callable(run_publish) - assert callable(_bq_table_columns) - _: Any = pytest # silence unused-import in the integration-only path diff --git a/jdluc/__tests__/test_tiling.py b/jdluc/__tests__/test_tiling.py new file mode 100644 index 0000000..14821f4 --- /dev/null +++ b/jdluc/__tests__/test_tiling.py @@ -0,0 +1,106 @@ +import pytest +import shapely + +from jdluc.tiling import ( + PARTITIONING_TO_IS_VALID_TILE_ID, + Partitioning, + get_lat_lon_for_tile_id, + get_tile_clusters, + get_tile_id_for_lat_lon, + iter_ten_degree_tile_id_for_geometry, + tiles_are_adjacent, +) + + +@pytest.mark.parametrize( + ("partitioning", "tile_id", "expected"), + ( + (Partitioning.ONE_DEGREE_TILE, "00N_000E", True), + (Partitioning.ONE_DEGREE_TILE, "01N_001E", True), + (Partitioning.ONE_DEGREE_TILE, "1N_1E", False), + (Partitioning.TEN_DEGREE_TILE, "00N_000E", True), + (Partitioning.TEN_DEGREE_TILE, "10N_010E", True), + (Partitioning.TEN_DEGREE_TILE, "05N_003E", False), + (Partitioning.WHOLE_WORLD, "world", True), + (Partitioning.WHOLE_WORLD, "CONUS", False), + (Partitioning.XYZ_MERCATOR_TILE, "000X_000Y_00Z", True), + (Partitioning.XYZ_MERCATOR_TILE, "001X_001Y_01Z", True), + (Partitioning.XYZ_MERCATOR_TILE, "1X_2Y_3Z", False), + ), +) +def test_partitioning_to_is_valid_tile_id( + partitioning: Partitioning, tile_id: str, expected: bool +) -> None: + assert PARTITIONING_TO_IS_VALID_TILE_ID[partitioning](tile_id) is expected + + +BOX_40N_080W = shapely.box(-80, 40, -70, 50) +PIXEL_45N_075W = shapely.Point(-75, +45).buffer(distance=1) +STREAK_SEVEN = shapely.LineString(((-75, +45), (-75, +35), (-85, +35))).buffer( + distance=1 +) + + +@pytest.mark.parametrize("lat", range(-90, +91, 10)) +@pytest.mark.parametrize("lon", range(-180, +181, 10)) +def test_lat_long_tile_id_roundtrip(lat: int, lon: int) -> None: + assert get_lat_lon_for_tile_id( + tile_id=get_tile_id_for_lat_lon(lat=lat, lon=lon) + ) == (lat, lon) + + +@pytest.mark.parametrize( + ("geometry", "expected"), + ( + (BOX_40N_080W, ("50N_080W",)), + (PIXEL_45N_075W, ("50N_080W",)), + (STREAK_SEVEN, ("40N_090W", "40N_080W", "50N_080W")), + ), +) +def test_iter_ten_degree_tile_id_for_geometry( + geometry: shapely.Polygon | shapely.MultiPolygon, expected: tuple[str, ...] +) -> None: + assert tuple(iter_ten_degree_tile_id_for_geometry(geometry=geometry)) == expected + + +@pytest.mark.parametrize( + ("left", "right", "expected"), + ( + ("00N_000W", "00N_000W", False), + ("00N_000W", "00S_000W", False), + ("00N_000W", "00N_000E", False), + ("00N_000W", "00S_000E", False), + ("00N_000W", "00N_010W", True), + ("00N_000W", "00S_010W", True), + ("00N_000W", "00N_010E", True), + ("00N_000W", "00S_010E", True), + ("00N_000W", "10N_000E", True), + ("00N_000W", "10N_000W", True), + ("00N_000W", "10S_000E", True), + ("00N_000W", "10S_000W", True), + # + ("60N_180W", "60N_170W", True), + ("60N_170W", "60N_180W", True), + ("60N_180W", "60N_170E", True), + ("60N_170E", "60N_180W", True), + ), +) +def test_is_adjacent(left: str, right: str, expected: bool) -> None: + assert tiles_are_adjacent(left=left, right=right) is expected + + +def test_get_tile_clusters() -> None: + result = get_tile_clusters( + tile_ids={"A", "B", "C", "E", "F", "H", "I"}, + is_adjacent=lambda l, r: abs(ord(l) - ord(r)) == 1, + edge_exclusions={("H", "I")}, + ) + assert len(result) == 4 + assert set("ABC") in result + assert set("EF") in result + assert { + "H", + } in result + assert { + "I", + } in result diff --git a/jdluc/__tests__/test_transform.py b/jdluc/__tests__/test_transform.py deleted file mode 100644 index 8b6c9d4..0000000 --- a/jdluc/__tests__/test_transform.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Unit tests for transform/transform.py (no GEE credentials required). - -The four-task integration is covered by the live-GEE test -test_transform_multistate_integration.py. -""" - -from contextlib import ExitStack -from typing import Any, cast -from unittest.mock import MagicMock, patch - -import pytest - -from jdluc.transform import transform as transform_module -from jdluc.transform.transform import ( - TransformError, - TransformResult, - TransformStage, - _load_fips_band_for_states, - _validate_inputs, - run_transform, -) - -# --------------------------------------------------------------------------- -# TransformError shape -# --------------------------------------------------------------------------- - - -def test_transform_error_carries_stage_and_message() -> None: - err = TransformError(TransformStage.LAND_USE.value, 'Export failed: Internal error') - assert err.failed_stage == 'LAND_USE' - assert err.error_message == 'Export failed: Internal error' - assert 'LAND_USE' in str(err) - assert 'Export failed: Internal error' in str(err) - - -def test_transform_error_stages_match_enum() -> None: - """Every TransformStage value can be embedded in a TransformError.""" - for stage in TransformStage: - err = TransformError(stage.value, 'oops') - assert err.failed_stage == stage.value - - -def test_transform_stage_enum_values() -> None: - """Public stage labels match the four-task design.""" - assert {s.name for s in TransformStage} == {'LAND_USE', 'EMISSIONS', 'TABLES'} - - -# --------------------------------------------------------------------------- -# _validate_inputs -# --------------------------------------------------------------------------- - - -def test_validate_inputs_rejects_empty_states() -> None: - with pytest.raises(ValueError, match='states must not be empty'): - _validate_inputs([], 'delaware') - - -def test_validate_inputs_rejects_empty_region_name() -> None: - with pytest.raises(ValueError, match='region_name must not be empty'): - _validate_inputs(['10'], '') - - -def test_validate_inputs_rejects_unknown_fips() -> None: - with pytest.raises(ValueError, match='Unknown state FIPS codes'): - _validate_inputs(['10', '99'], 'test') - - -def test_validate_inputs_excludes_alaska_and_hawaii() -> None: - # Alaska (02) and Hawaii (15) are explicitly not in STATE_FIPS_TO_NAME. - with pytest.raises(ValueError, match='Unknown state FIPS codes'): - _validate_inputs(['02'], 'alaska') - with pytest.raises(ValueError, match='Unknown state FIPS codes'): - _validate_inputs(['15'], 'hawaii') - - -def test_validate_inputs_accepts_valid_conus_fips() -> None: - _validate_inputs(['10', '19', '46'], 'test_region') - - -# --------------------------------------------------------------------------- -# run_transform shape (mocked GEE) -# --------------------------------------------------------------------------- -# The four-task orchestration is well-tested at integration level. These -# unit tests cover the shape of the public API: what `run_transform` -# returns, that `from_cache=True` falls out of all-cached runs, and that -# stage failures wrap into TransformError with the right stage label. - - -def _enter_baseline_patches(stack: ExitStack, **overrides: dict[str, object]) -> None: - """Enter the standard mock-everything patches on `stack`. - - Default patches make `run_transform` bypass GEE entirely: geometry + - counties resolve to MagicMocks, build/export functions return MagicMock - images / `None` (cache hit), and `compute_region_tables` returns a - fully-cached result. Override individual patches via `overrides` - keyword args, where the key is the attribute name relative to - `jdluc.transform.transform`. - """ - fake_tables = { - 'transitions_asset_id': 'projects/p/transitions', - 'crops_asset_id': 'projects/p/crops', - 'transitions_task': None, - 'crops_task': None, - } - defaults: dict[str, dict[str, object]] = { - 'get_multi_state_boundary': {'return_value': MagicMock(name='region_geometry')}, - '_load_fips_band_for_states': {'return_value': MagicMock(name='fips_band')}, - 'build_land_use_image': {'return_value': MagicMock()}, - 'export_land_use_asset': {'return_value': None}, - 'build_emissions_image': {'return_value': MagicMock()}, - 'export_emissions_asset': {'return_value': None}, - 'compute_region_tables': {'return_value': fake_tables}, - } - for attr, kwargs in defaults.items(): - merged = {**kwargs, **overrides.get(attr, {})} - # ``patch()`` overloads enumerate specific kwargs (return_value, - # side_effect, …) rather than accepting an arbitrary dict-splat, - # so we cast through Any to satisfy the call site. - stack.enter_context( - patch(f'jdluc.transform.transform.{attr}', **cast(dict[str, Any], merged)) - ) - - -def test_run_transform_all_cache_hit_yields_from_cache_true() -> None: - """When every export is a cache hit, result.from_cache=True.""" - with ExitStack() as stack: - _enter_baseline_patches(stack) - result = run_transform( - gcp_project='test', - states=['10'], - region_name='delaware', - force=False, - ) - assert isinstance(result, TransformResult) - assert result.from_cache is True - assert result.region_name == 'delaware' - assert result.transitions_table_id == 'projects/p/transitions' - assert result.crops_table_id == 'projects/p/crops' - - -def test_run_transform_wraps_land_use_failure_in_transform_error() -> None: - """Build failure on the LAND_USE stage raises TransformError(LAND_USE, ...).""" - with ExitStack() as stack: - _enter_baseline_patches( - stack, - build_land_use_image={ - 'side_effect': RuntimeError('synthetic land_use failure'), - 'return_value': None, - }, - ) - with pytest.raises(TransformError) as exc_info: - run_transform( - gcp_project='test', - states=['10'], - region_name='delaware', - force=False, - ) - assert exc_info.value.failed_stage == TransformStage.LAND_USE.value - assert 'synthetic land_use failure' in exc_info.value.error_message - - -def test_run_transform_wraps_tables_failure_in_transform_error() -> None: - """compute_region_tables failure raises TransformError(TABLES, ...).""" - with ExitStack() as stack: - _enter_baseline_patches( - stack, - compute_region_tables={ - 'side_effect': RuntimeError('synthetic tables failure'), - 'return_value': None, - }, - ) - with pytest.raises(TransformError) as exc_info: - run_transform( - gcp_project='test', - states=['10'], - region_name='delaware', - force=False, - ) - assert exc_info.value.failed_stage == TransformStage.TABLES.value - assert 'synthetic tables failure' in exc_info.value.error_message - - -# --------------------------------------------------------------------------- -# _load_fips_mask_for_states -# --------------------------------------------------------------------------- - - -def test_load_fips_band_single_state_filters_to_that_state_code() -> None: - """For one state, band = fips_label.updateMask(state_band == that_code).""" - with patch.object(transform_module, 'ee') as ee_mock: - fips_label = MagicMock(name='fips_label') - divided = MagicMock(name='divided') - state_band = MagicMock(name='state_band') - match_image = MagicMock(name='eq_match') - masked = MagicMock(name='masked') - renamed = MagicMock(name='renamed') - ee_mock.Image.return_value = fips_label - fips_label.divide.return_value = divided - divided.toInt.return_value = state_band - state_band.eq.return_value = match_image - fips_label.updateMask.return_value = masked - masked.rename.return_value = renamed - - result = _load_fips_band_for_states(['10']) - - assert result is renamed - fips_label.divide.assert_called_once_with(1000) - divided.toInt.assert_called_once_with() - state_band.eq.assert_called_once_with(10) - # No .Or() on the eq match for the single-state case. - match_image.Or.assert_not_called() - fips_label.updateMask.assert_called_once_with(match_image) - masked.rename.assert_called_once_with('county_fips') - - -def test_load_fips_band_multiple_states_chains_or_per_state_code() -> None: - """For N states, each subsequent state extends the mask via .Or().""" - with patch.object(transform_module, 'ee') as ee_mock: - fips_label = MagicMock(name='fips_label') - divided = MagicMock(name='divided') - state_band = MagicMock(name='state_band') - match_19 = MagicMock(name='eq_19') - match_31 = MagicMock(name='eq_31') - match_46 = MagicMock(name='eq_46') - or_19_31 = MagicMock(name='or_19_31') - or_19_31_46 = MagicMock(name='or_19_31_46') - masked = MagicMock(name='masked') - renamed = MagicMock(name='renamed') - - ee_mock.Image.return_value = fips_label - fips_label.divide.return_value = divided - divided.toInt.return_value = state_band - state_band.eq.side_effect = [match_19, match_31, match_46] - match_19.Or.return_value = or_19_31 - or_19_31.Or.return_value = or_19_31_46 - fips_label.updateMask.return_value = masked - masked.rename.return_value = renamed - - result = _load_fips_band_for_states(['19', '31', '46']) - - assert result is renamed - eq_calls = [c.args[0] for c in state_band.eq.call_args_list] - assert eq_calls == [19, 31, 46] - # First state's match is or'd with the second, then the third. - match_19.Or.assert_called_once_with(match_31) - or_19_31.Or.assert_called_once_with(match_46) - fips_label.updateMask.assert_called_once_with(or_19_31_46) - - -def test_load_fips_band_uses_canonical_county_fips_asset() -> None: - """The asset path is the canonical GEE_COUNTY_FIPS_LABEL constant.""" - from jdluc.utils.constants import GEE_COUNTY_FIPS_LABEL - - with patch.object(transform_module, 'ee') as ee_mock: - # Set up a minimal chain so the function returns without error. - fips_label = MagicMock() - ee_mock.Image.return_value = fips_label - fips_label.divide.return_value.toInt.return_value.eq.return_value = MagicMock() - fips_label.updateMask.return_value.rename.return_value = MagicMock() - - _load_fips_band_for_states(['19']) - - ee_mock.Image.assert_called_once_with(GEE_COUNTY_FIPS_LABEL) diff --git a/jdluc/__tests__/test_transform_emissions_integration.py b/jdluc/__tests__/test_transform_emissions_integration.py deleted file mode 100644 index 21d7a18..0000000 --- a/jdluc/__tests__/test_transform_emissions_integration.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Integration tests for the emissions raster (Delaware fixture). - -Asserts that the exported emissions raster has the correct band structure, -dtypes, grid alignment, non-negativity, and mask invariants. Requires a -live GEE connection and previously exported Delaware emissions and -land_use assets (via cli.py). - -All tests are marked @pytest.mark.integration and excluded from CI by -default (run with: ``pytest -m integration``). -""" - -from typing import Any - -import ee -import pytest - -from jdluc.transform.land_use import LAND_USE_BAND_NAMES -from jdluc.utils.asset_management import list_assets_matching -from jdluc.utils.constants import ( - EMISSIONS_BAND_NAMES, - EMISSIVE_ENCODED_CODES, - GCP_PROJECT, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - GLAD_EPOCH_PAIRS, - PEATLAND_DRAINAGE_ENCODED_CODES, - PEATLAND_E_LM_TCO2E_HA_YR, -) -from jdluc.utils.states import get_multi_state_boundary -from jdluc.utils.version import compute_transform_version - -from .conftest import DELAWARE_FIPS - -REGION = 'delaware' - -# scale= for Delaware-wide reduceRegion calls. 30m matches GLAD pixel size; -# Delaware at 30m is ~45M pixels, well within maxPixels=1e9. -_REDUCE_SCALE = 30 -_REDUCE_MAX_PIXELS = int(1e9) - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope='module') -def gee_init() -> None: - """Initialize GEE once for this test module.""" - from jdluc.utils.gee import initialize_gee - - initialize_gee(GCP_PROJECT) - - -@pytest.fixture(scope='module') -def _current_version(gee_init: None) -> str: - return compute_transform_version() - - -@pytest.fixture(scope='module') -def emissions_asset_id(_current_version: str) -> str: - """Resolve the current-version Delaware emissions asset ID. - - Skips the entire module if no matching asset exists. - """ - expected_suffix = f'emissions_{REGION}_{_current_version}' - for asset_id in list_assets_matching(f'emissions_{REGION}'): - if asset_id.endswith(expected_suffix): - return asset_id - pytest.skip( - f'No emissions asset for {REGION} v{_current_version}. ' 'Run cli.py first.' - ) - - -@pytest.fixture(scope='module') -def land_use_asset_id(_current_version: str) -> str: - """Resolve the current-version Delaware land_use asset ID.""" - expected_suffix = f'land_use_{REGION}_{_current_version}' - for asset_id in list_assets_matching(f'land_use_{REGION}'): - if asset_id.endswith(expected_suffix): - return asset_id - pytest.skip( - f'No land_use asset for {REGION} v{_current_version}. ' 'Run cli.py first.' - ) - - -@pytest.fixture(scope='module') -def emissions_asset(emissions_asset_id: str) -> Any: - return ee.Image(emissions_asset_id) - - -@pytest.fixture(scope='module') -def emissions_info(emissions_asset: Any) -> dict[str, Any]: - return emissions_asset.getInfo() - - -@pytest.fixture(scope='module') -def land_use_asset(land_use_asset_id: str) -> Any: - return ee.Image(land_use_asset_id) - - -@pytest.fixture(scope='module') -def delaware_geometry(gee_init: None) -> Any: - return get_multi_state_boundary([DELAWARE_FIPS]) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_asset_exists(emissions_asset_id: str, _current_version: str) -> None: - """The current-version emissions asset exists under the expected name.""" - assert emissions_asset_id.endswith(f'emissions_{REGION}_{_current_version}') - - -@pytest.mark.integration -def test_band_count_and_names(emissions_info: dict[str, Any]) -> None: - """Asset has exactly the 11 expected bands in canonical order.""" - band_names = [b['id'] for b in emissions_info['bands']] - assert band_names == EMISSIONS_BAND_NAMES - - -@pytest.mark.integration -def test_band_dtypes(emissions_info: dict[str, Any]) -> None: - """Every band is float32.""" - for band in emissions_info['bands']: - dt = band['data_type'] - assert ( - dt['precision'] == 'float' - ), f"Band {band['id']}: expected float precision, got {dt['precision']}" - - -@pytest.mark.integration -def test_grid_alignment(emissions_info: dict[str, Any]) -> None: - """Every band is pinned to the GLAD CRS and pixel grid. - - GEE stores the crs_transform with a localized origin (the asset's - bounding box) rather than the global origin (-180, 90). So we check: - 1. CRS is EPSG:4326 - 2. Pixel size matches GLAD_CRS_TRANSFORM exactly - 3. Origin is phase-aligned with the GLAD global grid - """ - pixel_x = GLAD_CRS_TRANSFORM[0] - pixel_y = GLAD_CRS_TRANSFORM[4] - global_x = GLAD_CRS_TRANSFORM[2] - global_y = GLAD_CRS_TRANSFORM[5] - - for band in emissions_info['bands']: - bid = band['id'] - assert band['crs'] == GLAD_CRS, f'{bid}: crs={band["crs"]}' - t = band['crs_transform'] - assert abs(t[0] - pixel_x) < 1e-12, f'{bid}: pixel_x={t[0]}' - assert abs(t[4] - pixel_y) < 1e-12, f'{bid}: pixel_y={t[4]}' - assert t[1] == 0 and t[3] == 0, f'{bid}: unexpected rotation' - offset_x = (t[2] - global_x) / pixel_x - offset_y = (t[5] - global_y) / pixel_y - assert ( - abs(offset_x - round(offset_x)) < 1e-6 - ), f'{bid}: origin_x {t[2]} off GLAD phase' - assert ( - abs(offset_y - round(offset_y)) < 1e-6 - ), f'{bid}: origin_y {t[5]} off GLAD phase' - - -def _region_min(image: Any, band: str, geometry: Any) -> float: - return ( - image.select(band) - .reduceRegion( - reducer=ee.Reducer.min(), - geometry=geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .get(band) - .getInfo() - ) - - -def _region_sum(image: Any, band: str, geometry: Any) -> float: - return ( - image.select(band) - .reduceRegion( - reducer=ee.Reducer.sum(), - geometry=geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .get(band) - .getInfo() - ) - - -@pytest.mark.integration -@pytest.mark.parametrize('band_name', EMISSIONS_BAND_NAMES) -def test_non_negativity( - emissions_asset: Any, delaware_geometry: Any, band_name: str -) -> None: - """Every band is non-negative (float32 epsilon tolerance).""" - min_val = _region_min(emissions_asset, band_name, delaware_geometry) - assert ( - min_val is not None and min_val >= -1e-6 - ), f'{band_name}: min={min_val} violates non-negativity' - - -@pytest.mark.integration -def test_allocated_luc_at_most_sum_of_epochs( - emissions_asset: Any, delaware_geometry: Any -) -> None: - """sum(allocated_luc_emissions_2020) ≤ Σ sum(luc_emissions_{epoch}) + slack. - - Slack is 1 tCO₂, well below Delaware's per-epoch sums (millions). - """ - allocated = _region_sum( - emissions_asset, 'allocated_luc_emissions_2020', delaware_geometry - ) - per_epoch = sum( - _region_sum(emissions_asset, f'luc_emissions_{a}_{b}', delaware_geometry) - for (a, b) in GLAD_EPOCH_PAIRS - ) - assert ( - allocated <= per_epoch + 1.0 - ), f'allocated_luc={allocated} > sum_per_epoch={per_epoch}' - - -@pytest.mark.integration -def test_allocated_peatland_at_most_sum( - emissions_asset: Any, delaware_geometry: Any -) -> None: - """allocated_peatland ≤ Σ peatland_conversion + peatland_occupation + slack.""" - allocated = _region_sum( - emissions_asset, 'allocated_peatland_emissions_2020', delaware_geometry - ) - per_epoch = sum( - _region_sum(emissions_asset, f'peatland_conversion_{a}_{b}', delaware_geometry) - for (a, b) in GLAD_EPOCH_PAIRS - ) - occupation = _region_sum( - emissions_asset, 'peatland_occupation_2020', delaware_geometry - ) - assert allocated <= per_epoch + occupation + 1.0, ( - f'allocated_peatland={allocated} > ' - f'per_epoch+occupation={per_epoch + occupation}' - ) - - -@pytest.mark.integration -@pytest.mark.parametrize('epoch', GLAD_EPOCH_PAIRS) -def test_luc_emissions_gated_on_emissive_mask( - emissions_asset: Any, - land_use_asset: Any, - delaware_geometry: Any, - epoch: tuple[int, int], -) -> None: - """luc_emissions_{epoch} is zero on pixels whose transition is not in the - 9-pair emissive vocabulary.""" - a, b = epoch - transition = land_use_asset.select(f'transitions_{a}_{b}') - is_emissive = transition.remap( - EMISSIVE_ENCODED_CODES, [1] * len(EMISSIVE_ENCODED_CODES), defaultValue=0 - ) - non_emissive = is_emissive.Not() - leaked = ( - emissions_asset.select(f'luc_emissions_{a}_{b}') - .multiply(non_emissive) - .reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .values() - .getInfo()[0] - ) - assert ( - leaked == 0 - ), f'luc_emissions_{a}_{b} has {leaked} tCO₂ on non-emissive pixels' - - -@pytest.mark.integration -@pytest.mark.parametrize('epoch', GLAD_EPOCH_PAIRS) -def test_peatland_conversion_gated_on_drainage_mask( - emissions_asset: Any, - land_use_asset: Any, - delaware_geometry: Any, - epoch: tuple[int, int], -) -> None: - """peatland_conversion_{epoch} is zero on pixels whose destination is not - cropland or built_up — specifically verifies forest_to_short_veg on - peatland does NOT produce a conversion pulse (Design decision 5).""" - a, b = epoch - transition = land_use_asset.select(f'transitions_{a}_{b}') - is_drainage = transition.remap( - PEATLAND_DRAINAGE_ENCODED_CODES, - [1] * len(PEATLAND_DRAINAGE_ENCODED_CODES), - defaultValue=0, - ) - non_drainage = is_drainage.Not() - leaked = ( - emissions_asset.select(f'peatland_conversion_{a}_{b}') - .multiply(non_drainage) - .reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .values() - .getInfo()[0] - ) - assert ( - leaked == 0 - ), f'peatland_conversion_{a}_{b} has {leaked} tCO₂ on non-drainage pixels' - - -@pytest.mark.integration -def test_peatland_occupation_gated_on_peatland_and_cropland( - emissions_asset: Any, land_use_asset: Any, delaware_geometry: Any -) -> None: - """peatland_occupation_2020 is zero outside (is_peatland & crops_2020>0).""" - is_peatland = land_use_asset.select('is_peatland').gt(0) - is_cropland = land_use_asset.select('crops_2020').gt(0) - outside_mask = is_peatland.multiply(is_cropland).Not() - leaked = ( - emissions_asset.select('peatland_occupation_2020') - .multiply(outside_mask) - .reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .values() - .getInfo()[0] - ) - assert ( - leaked == 0 - ), f'peatland_occupation_2020 has {leaked} tCO₂ outside peatland∩cropland' - - -@pytest.mark.integration -def test_peatland_occupation_uniform_per_pixel( - emissions_asset: Any, delaware_geometry: Any -) -> None: - """Every non-zero peatland_occupation pixel equals E_LM × pixel_area_ha - (up to 1e-3 tCO₂e). Skips if Delaware has no peatland∩cropland pixels.""" - occ = emissions_asset.select('peatland_occupation_2020') - area_ha = ( - ee.Image.pixelArea() - .divide(1e4) - .reproject(crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM) - ) - expected = area_ha.multiply(PEATLAND_E_LM_TCO2E_HA_YR) - # Only compare on pixels where occupation is non-zero - on_mask = occ.gt(0) - max_count = ( - on_mask.reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .values() - .getInfo()[0] - ) - if max_count == 0: - pytest.skip('No peatland∩cropland pixels in Delaware') - - diff = occ.subtract(expected).abs().updateMask(on_mask) - max_diff = ( - diff.reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .values() - .getInfo()[0] - ) - assert max_diff < 1e-3, ( - f'peatland_occupation non-uniform: max |actual - E_LM*area_ha| = ' f'{max_diff}' - ) - - -@pytest.mark.integration -def test_epoch_sensitivity(emissions_asset: Any, delaware_geometry: Any) -> None: - """The four luc_emissions_{epoch} regional sums are not all identical.""" - sums = [ - _region_sum(emissions_asset, f'luc_emissions_{a}_{b}', delaware_geometry) - for (a, b) in GLAD_EPOCH_PAIRS - ] - assert ( - max(sums) - min(sums) > 1e-3 - ), f'luc_emissions regional sums identical across epochs: {sums}' - - -@pytest.mark.integration -def test_transition_bands_order_check( - land_use_asset: Any, -) -> None: - """Defensive: the land_use asset's bands still match LAND_USE_BAND_NAMES. - - If a future change ever reorders the land_use schema, the emissions - tests that rely on transitions_{epoch} / is_peatland / crops_2020 band - names break too — this test surfaces it loudly. - """ - actual = land_use_asset.bandNames().getInfo() - assert actual == LAND_USE_BAND_NAMES diff --git a/jdluc/__tests__/test_transform_land_use_integration.py b/jdluc/__tests__/test_transform_land_use_integration.py deleted file mode 100644 index 4f82704..0000000 --- a/jdluc/__tests__/test_transform_land_use_integration.py +++ /dev/null @@ -1,306 +0,0 @@ -"""Integration tests for the land_use raster (Delaware fixture). - -Asserts that the exported land_use raster has the correct band structure, -dtypes, grid alignment, and plausible pixel values. Requires a live GEE -connection and a previously exported Delaware land_use asset (via -cli.py or the fixture below). - -All tests are marked @pytest.mark.integration and excluded from CI by -default (run with: ``pytest -m integration``). -""" - -from typing import Any - -import ee -import pytest - -from jdluc.transform.land_use import LAND_USE_BAND_NAMES -from jdluc.utils.asset_management import list_assets_matching -from jdluc.utils.constants import ( - GCP_PROJECT, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - LAND_USE_CATEGORIES, -) -from jdluc.utils.states import get_multi_state_boundary -from jdluc.utils.version import compute_transform_version - -from .conftest import DELAWARE_FIPS - -# GCP project for Earth Engine -REGION = 'delaware' - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope='module') -def gee_init() -> None: - """Initialize GEE once for this test module.""" - from jdluc.utils.gee import initialize_gee - - initialize_gee(GCP_PROJECT) - - -@pytest.fixture(scope='module') -def land_use_asset_id(gee_init: None) -> str: - """Resolve the current-version Delaware land_use asset ID. - - Skips the entire module if no matching asset exists. - """ - version = compute_transform_version() - matches = list_assets_matching(f'land_use_{REGION}') - expected_suffix = f'land_use_{REGION}_{version}' - for asset_id in matches: - if asset_id.endswith(expected_suffix): - return asset_id - pytest.skip(f'No land_use asset for {REGION} v{version}. Run cli.py first.') - - -@pytest.fixture(scope='module') -def asset_info(land_use_asset_id: str) -> dict[str, Any]: - """Load the full asset metadata via .getInfo() once.""" - return ee.Image(land_use_asset_id).getInfo() - - -@pytest.fixture(scope='module') -def asset_image(land_use_asset_id: str) -> Any: - """Load the asset as an ee.Image for server-side queries.""" - return ee.Image(land_use_asset_id) - - -@pytest.fixture(scope='module') -def delaware_geometry(gee_init: None) -> Any: - """Delaware state boundary geometry.""" - return get_multi_state_boundary([DELAWARE_FIPS]) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_asset_exists(land_use_asset_id: str) -> None: - """The current-version land_use asset exists in GEE.""" - assert land_use_asset_id is not None - - -@pytest.mark.integration -def test_band_count_and_names(asset_info: dict[str, Any]) -> None: - """Asset has exactly the 6 expected bands in the correct order.""" - band_names = [b['id'] for b in asset_info['bands']] - assert band_names == LAND_USE_BAND_NAMES - - -@pytest.mark.integration -def test_band_dtypes(asset_info: dict[str, Any]) -> None: - """Every band is uint8.""" - for band in asset_info['bands']: - data_type = band['data_type'] - # GEE reports dtype as a dict with 'precision' and 'min'/'max' - assert ( - data_type['precision'] == 'int' - ), f"Band {band['id']}: expected int precision, got {data_type['precision']}" - assert data_type['min'] == 0, f"Band {band['id']}: min != 0" - assert data_type['max'] == 255, f"Band {band['id']}: max != 255" - - -@pytest.mark.integration -def test_grid_alignment(asset_info: dict[str, Any]) -> None: - """Every band is pinned to the GLAD CRS and pixel grid. - - GEE stores the crs_transform with a localized origin (the asset's - bounding box) rather than the global origin (-180, 90). So we check: - 1. CRS is EPSG:4326 - 2. Pixel size matches GLAD_CRS_TRANSFORM exactly - 3. Origin is phase-aligned with the GLAD global grid (offsets are - integer multiples of the pixel size from the global origin) - """ - pixel_size_x = GLAD_CRS_TRANSFORM[0] # 0.00025 - pixel_size_y = GLAD_CRS_TRANSFORM[4] # -0.00025 - global_origin_x = GLAD_CRS_TRANSFORM[2] # -180 - global_origin_y = GLAD_CRS_TRANSFORM[5] # 90 - - for band in asset_info['bands']: - bid = band['id'] - assert ( - band['crs'] == GLAD_CRS - ), f"Band {bid}: expected {GLAD_CRS}, got {band['crs']}" - - t = band['crs_transform'] - # Pixel sizes must match exactly - assert ( - abs(t[0] - pixel_size_x) < 1e-12 - ), f"Band {bid}: pixel_size_x = {t[0]}, expected {pixel_size_x}" - assert ( - abs(t[4] - pixel_size_y) < 1e-12 - ), f"Band {bid}: pixel_size_y = {t[4]}, expected {pixel_size_y}" - # No rotation/shear - assert ( - t[1] == 0 and t[3] == 0 - ), f"Band {bid}: unexpected rotation in crs_transform" - # Origin must be phase-aligned with the GLAD global grid - offset_x = (t[2] - global_origin_x) / pixel_size_x - offset_y = (t[5] - global_origin_y) / pixel_size_y - assert abs(offset_x - round(offset_x)) < 1e-6, ( - f"Band {bid}: origin_x {t[2]} not phase-aligned with GLAD grid " - f"(offset {offset_x} pixels from global origin)" - ) - assert abs(offset_y - round(offset_y)) < 1e-6, ( - f"Band {bid}: origin_y {t[5]} not phase-aligned with GLAD grid " - f"(offset {offset_y} pixels from global origin)" - ) - - -@pytest.mark.integration -def test_is_peatland_nonzero(asset_image: Any, delaware_geometry: Any) -> None: - """is_peatland band has at least one nonzero pixel in Delaware.""" - max_val = ( - asset_image.select('is_peatland') - .reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ) - .getInfo() - ) - assert ( - max_val['is_peatland'] == 1 - ), 'Expected peatland pixels in Delaware (coastal plain), got none' - - -@pytest.mark.integration -def test_crops_2020_distribution(asset_image: Any, delaware_geometry: Any) -> None: - """crops_2020 band contains corn (1), soybeans (5), and wheat codes.""" - histogram = ( - asset_image.select('crops_2020') - .reduceRegion( - reducer=ee.Reducer.frequencyHistogram(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ) - .getInfo() - ) - hist = histogram['crops_2020'] - - # CDL codes: 1=corn, 5=soybeans, 22/23/24=wheat varieties - assert '1' in hist, 'No corn (CDL code 1) in Delaware crops_2020' - assert '5' in hist, 'No soybeans (CDL code 5) in Delaware crops_2020' - - wheat_codes = ['22', '23', '24'] - wheat_present = any(c in hist for c in wheat_codes) - assert wheat_present, 'No wheat codes (22/23/24) in Delaware crops_2020' - - # Non-cropland (code 0) should dominate but not be 100% - total_nonzero = sum(v for k, v in hist.items() if k != '0') - assert total_nonzero > 0, 'crops_2020 has no cropland pixels' - - -@pytest.mark.integration -def test_transition_bands_nonzero(asset_image: Any, delaware_geometry: Any) -> None: - """Every transition band has at least one non-zero pixel.""" - transition_bands = LAND_USE_BAND_NAMES[:4] - for band_name in transition_bands: - max_val = ( - asset_image.select(band_name) - .reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ) - .getInfo() - ) - assert max_val[band_name] > 0, f'{band_name} has no transitions (all zeros)' - - -@pytest.mark.integration -def test_no_self_transitions(asset_image: Any, delaware_geometry: Any) -> None: - """No transition band pixel encodes from==to (which would be a bug).""" - # Self-transition codes: (i<<4)|i for i in 1..9 - # encode_transition(i,i) returns 0 by construction, so we check for the - # raw bit pattern (i<<4)|i which would indicate a bug in the encoder. - self_code_images = [] - for i in [c.code for c in LAND_USE_CATEGORIES.values()]: - raw_self = (i << 4) | i # e.g. 0x11, 0x22, ... - self_code_images.append(raw_self) - - transition_bands = LAND_USE_BAND_NAMES[:4] - for band_name in transition_bands: - band = asset_image.select(band_name) - # Check if any pixel has a self-transition code - has_self = band.eq(self_code_images[0]) - for code in self_code_images[1:]: - has_self = has_self.Or(band.eq(code)) - - max_self = has_self.reduceRegion( - reducer=ee.Reducer.max(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ).getInfo() - # The result key is 'category' from the .eq() output - result_key = list(max_self.keys())[0] - assert ( - max_self[result_key] == 0 - ), f'{band_name} has pixels with from==to self-transition codes' - - -@pytest.mark.integration -def test_classification_coverage(asset_image: Any, delaware_geometry: Any) -> None: - """At least 99% of pixels in any transition band have valid categories. - - Decodes from/to from a transition band and checks that unclassified (0) - pixels are rare. Only checks non-zero (transitioned) pixels since zero - means "no transition" which is valid. - """ - from jdluc.utils.transitions import ( - decode_from_image, - decode_to_image, - ) - - # Use the first transition band as representative - band = asset_image.select(LAND_USE_BAND_NAMES[0]) - - # Count total non-zero (transitioned) pixels - transitioned = band.gt(0) - total_transitioned = transitioned.reduceRegion( - reducer=ee.Reducer.sum(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ).getInfo() - total_key = list(total_transitioned.keys())[0] - total_count = total_transitioned[total_key] - - if total_count == 0: - pytest.skip('No transitioned pixels to check coverage on') - - # Among transitioned pixels, check from and to codes are in 1-9 - from_code = decode_from_image(band) - to_code = decode_to_image(band) - - # Invalid = from or to is 0 on a transitioned pixel - invalid_from = from_code.eq(0).And(transitioned) - invalid_to = to_code.eq(0).And(transitioned) - invalid = invalid_from.Or(invalid_to) - - invalid_count_info = invalid.reduceRegion( - reducer=ee.Reducer.sum(), - geometry=delaware_geometry, - scale=30, - maxPixels=int(1e9), - ).getInfo() - invalid_key = list(invalid_count_info.keys())[0] - invalid_count = invalid_count_info[invalid_key] - - pct_invalid = (invalid_count / total_count) * 100 if total_count > 0 else 0 - assert pct_invalid < 1.0, ( - f'{pct_invalid:.2f}% of transitioned pixels have unclassified ' - f'(0) from/to codes — expected < 1%' - ) diff --git a/jdluc/__tests__/test_transform_multistate_integration.py b/jdluc/__tests__/test_transform_multistate_integration.py deleted file mode 100644 index 5da6365..0000000 --- a/jdluc/__tests__/test_transform_multistate_integration.py +++ /dev/null @@ -1,345 +0,0 @@ -"""Integration tests for multi-state transform runs (Iowa + Nebraska + SD). - -Exercises the per-state phase tracker + consolidation against -pre-exported per-state and consolidated assets for the -`great_plains_test` region (FIPS 19, 31, 46). Requires a live GEE -connection and a prior successful run via: - - uv run python jdluc/cli.py --region great_plains_test - -All tests are marked @pytest.mark.integration and excluded from CI by -default (run with: ``pytest -m integration``). - -The failure-injection + resume behavior is covered at the -unit-test level in test_transform.py via mocked task handles; -reproducing it here against live GEE is a manual procedure documented -below (§ Manual failure-injection procedure). -""" - -import math -from typing import Any - -import ee -import pytest - -from jdluc.utils.asset_management import list_assets_matching -from jdluc.utils.constants import ( - CROPS_TABLE_COLUMNS, - GCP_PROJECT, - STATE_FIPS_TO_NAME, - TRANSITIONS_TABLE_COLUMNS, -) -from jdluc.utils.states import ( - get_multi_state_boundary, -) -from jdluc.utils.version import compute_transform_version - -# Multi-state test region: Iowa + Nebraska + South Dakota. -_STATES: list[str] = ['19', '31', '46'] -_REGION_NAME = 'great_plains_test' - -_ROUND_TRIP_SLACK_TCO2 = 10.0 # Larger than Delaware's ~1 because IA alone -# has ~100x more emissions; 10 tCO2 is still float32 noise. - -_REDUCE_SCALE = 30 -_REDUCE_MAX_PIXELS = int(1e12) # 3 states, ~6B pixels total - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope='module') -def gee_init() -> None: - from jdluc.utils.gee import initialize_gee - - initialize_gee(GCP_PROJECT) - - -@pytest.fixture(scope='module') -def _current_version(gee_init: None) -> str: - return compute_transform_version() - - -def _resolve_asset_id(prefix: str, region_or_state: str, version: str) -> str: - """Find an asset ID by (prefix_region_version) suffix; skip if missing.""" - expected_suffix = f'{prefix}_{region_or_state}_{version}' - for asset_id in list_assets_matching(f'{prefix}_{region_or_state}'): - if asset_id.endswith(expected_suffix): - return asset_id - pytest.skip( - f'No {prefix} asset for {region_or_state} v{version}. ' - f'Run cli.py --region {_REGION_NAME} first.' - ) - - -@pytest.fixture(scope='module') -def consolidated_asset_ids(_current_version: str) -> dict[str, str]: - """IDs for the four consolidated region-level assets.""" - return { - kind: _resolve_asset_id(kind, _REGION_NAME, _current_version) - for kind in ('land_use', 'emissions', 'transitions', 'crops') - } - - -@pytest.fixture(scope='module') -def per_state_asset_ids(_current_version: str) -> dict[str, dict[str, str]]: - """{state_fips: {asset_kind: asset_id}} for every state and kind.""" - out: dict[str, dict[str, str]] = {} - for fips in _STATES: - state_name = STATE_FIPS_TO_NAME[fips] - out[fips] = { - kind: _resolve_asset_id(kind, state_name, _current_version) - for kind in ('land_use', 'emissions', 'transitions', 'crops') - } - return out - - -@pytest.fixture(scope='module') -def region_geometry(gee_init: None) -> Any: - return get_multi_state_boundary(_STATES) - - -def _reduce_band_sum(asset_id: str, band: str, geometry: Any) -> float: - """Sum one band of an image asset over a geometry.""" - value = ( - ee.Image(asset_id) - .select(band) - .reduceRegion( - reducer=ee.Reducer.sum(), - geometry=geometry, - scale=_REDUCE_SCALE, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .get(band) - .getInfo() - ) - return float(value) if value is not None else 0.0 - - -def _table_size(asset_id: str) -> int: - return int(ee.FeatureCollection(asset_id).size().getInfo()) - - -def _table_column_sum(asset_id: str, column: str) -> float: - value = ee.FeatureCollection(asset_id).aggregate_sum(column).getInfo() - return float(value) if value is not None else 0.0 - - -# --------------------------------------------------------------------------- -# Multi-state happy path -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_all_per_state_assets_exist( - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """Each of 3 states has all 4 per-state assets at the current version.""" - for fips in _STATES: - for kind in ('land_use', 'emissions', 'transitions', 'crops'): - assert per_state_asset_ids[fips][ - kind - ], f'missing per-state {kind} for fips {fips}' - - -@pytest.mark.integration -def test_all_consolidated_assets_exist( - consolidated_asset_ids: dict[str, str], -) -> None: - """All 4 consolidated assets exist under the great_plains_test label.""" - for kind in ('land_use', 'emissions', 'transitions', 'crops'): - assert consolidated_asset_ids[kind].endswith( - f'{kind}_{_REGION_NAME}_{compute_transform_version()}' - ), f'consolidated {kind} asset ID mismatch' - - -@pytest.mark.integration -def test_per_state_emissions_non_negative( - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """Each state's allocated_luc_emissions_2020 band totals to >= 0.""" - for fips in _STATES: - total = _reduce_band_sum( - per_state_asset_ids[fips]['emissions'], - 'allocated_luc_emissions_2020', - get_multi_state_boundary([fips]), - ) - assert total >= 0.0, f'negative allocated LUC total in state {fips}' - - -@pytest.mark.integration -def test_per_state_transitions_have_rows( - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """Each state's transitions table has >= 20 rows (meaningful coverage).""" - for fips in _STATES: - size = _table_size(per_state_asset_ids[fips]['transitions']) - assert size >= 20, f'state {fips} transitions table has only {size} rows' - - -@pytest.mark.integration -def test_per_state_crops_have_three_groups( - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """Each state's crops table covers corn, soybeans, and wheat.""" - for fips in _STATES: - fc = ee.FeatureCollection(per_state_asset_ids[fips]['crops']) - groups = set(fc.aggregate_array('crop_group').getInfo()) - assert groups == { - 'corn', - 'soybeans', - 'wheat', - }, f'state {fips} crops table missing groups: {groups}' - - -# --------------------------------------------------------------------------- -# Consolidated / per-state equivalence -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_consolidated_emissions_matches_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], - region_geometry: Any, -) -> None: - """Σ per-state allocated LUC = consolidated allocated LUC within slack.""" - consolidated_sum = _reduce_band_sum( - consolidated_asset_ids['emissions'], - 'allocated_luc_emissions_2020', - region_geometry, - ) - per_state_sum = sum( - _reduce_band_sum( - per_state_asset_ids[fips]['emissions'], - 'allocated_luc_emissions_2020', - get_multi_state_boundary([fips]), - ) - for fips in _STATES - ) - # Larger slack than Delaware: 3 states sum up to ~1e6 tCO2, 10 is noise. - assert ( - abs(consolidated_sum - per_state_sum) <= _ROUND_TRIP_SLACK_TCO2 - ), f'consolidated={consolidated_sum}, per-state sum={per_state_sum}' - - -@pytest.mark.integration -def test_consolidated_peatland_matches_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], - region_geometry: Any, -) -> None: - """Same round-trip check on the peatland band.""" - consolidated_sum = _reduce_band_sum( - consolidated_asset_ids['emissions'], - 'allocated_peatland_emissions_2020', - region_geometry, - ) - per_state_sum = sum( - _reduce_band_sum( - per_state_asset_ids[fips]['emissions'], - 'allocated_peatland_emissions_2020', - get_multi_state_boundary([fips]), - ) - for fips in _STATES - ) - assert abs(consolidated_sum - per_state_sum) <= _ROUND_TRIP_SLACK_TCO2 - - -@pytest.mark.integration -def test_consolidated_transitions_row_count_equals_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """`FC.size()` on consolidated = Σ `FC.size()` on per-state tables.""" - consolidated_size = _table_size(consolidated_asset_ids['transitions']) - per_state_size = sum( - _table_size(per_state_asset_ids[fips]['transitions']) for fips in _STATES - ) - assert ( - consolidated_size == per_state_size - ), f'consolidated={consolidated_size}, per-state sum={per_state_size}' - - -@pytest.mark.integration -def test_consolidated_crops_row_count_equals_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """`FC.size()` round-trip on the crops tables.""" - consolidated_size = _table_size(consolidated_asset_ids['crops']) - per_state_size = sum( - _table_size(per_state_asset_ids[fips]['crops']) for fips in _STATES - ) - assert consolidated_size == per_state_size - - -@pytest.mark.integration -def test_consolidated_total_emissions_matches_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """total_emissions_tco2 aggregate sum across transitions tables.""" - consolidated_sum = _table_column_sum( - consolidated_asset_ids['transitions'], 'total_emissions_tco2' - ) - per_state_sum = sum( - _table_column_sum( - per_state_asset_ids[fips]['transitions'], 'total_emissions_tco2' - ) - for fips in _STATES - ) - assert math.isclose( - consolidated_sum, per_state_sum, abs_tol=_ROUND_TRIP_SLACK_TCO2 - ), f'consolidated={consolidated_sum}, per-state sum={per_state_sum}' - - -@pytest.mark.integration -def test_consolidated_allocated_crops_matches_per_state_sum( - consolidated_asset_ids: dict[str, str], - per_state_asset_ids: dict[str, dict[str, str]], -) -> None: - """total_allocated_emissions_tco2 aggregate sum across crops tables.""" - consolidated_sum = _table_column_sum( - consolidated_asset_ids['crops'], 'total_allocated_emissions_tco2' - ) - per_state_sum = sum( - _table_column_sum( - per_state_asset_ids[fips]['crops'], 'total_allocated_emissions_tco2' - ) - for fips in _STATES - ) - assert math.isclose( - consolidated_sum, per_state_sum, abs_tol=_ROUND_TRIP_SLACK_TCO2 - ), f'consolidated={consolidated_sum}, per-state sum={per_state_sum}' - - -@pytest.mark.integration -def test_consolidated_schema_matches_spec( - consolidated_asset_ids: dict[str, str], -) -> None: - """Consolidated transitions and crops tables carry the canonical columns.""" - trans_fc = ee.FeatureCollection(consolidated_asset_ids['transitions']) - first_trans = trans_fc.first().toDictionary().getInfo() - assert set(first_trans.keys()) == set(TRANSITIONS_TABLE_COLUMNS) - - crops_fc = ee.FeatureCollection(consolidated_asset_ids['crops']) - first_crops = crops_fc.first().toDictionary().getInfo() - assert set(first_crops.keys()) == set(CROPS_TABLE_COLUMNS) - - -@pytest.mark.integration -def test_counties_span_all_three_states( - consolidated_asset_ids: dict[str, str], -) -> None: - """Every county_fips in consolidated transitions starts with 19/31/46.""" - fc = ee.FeatureCollection(consolidated_asset_ids['transitions']) - county_fips_list = set(fc.aggregate_array('county_fips').getInfo()) - state_prefixes = {str(c)[:2] for c in county_fips_list} - assert state_prefixes == { - '19', - '31', - '46', - }, f'unexpected state prefixes in consolidated transitions: {state_prefixes}' diff --git a/jdluc/__tests__/test_transform_summary_tables.py b/jdluc/__tests__/test_transform_summary_tables.py deleted file mode 100644 index a8371f8..0000000 --- a/jdluc/__tests__/test_transform_summary_tables.py +++ /dev/null @@ -1,343 +0,0 @@ -"""Unit tests for transform/summary_tables.py (no GEE credentials required). - -Integration tests that hit live GEE live in -test_transform_summary_tables_integration.py. -""" - -import math -from unittest.mock import MagicMock, patch - -from jdluc.transform import summary_tables as st -from jdluc.transform.summary_tables import ( - _FOREST_ENCODED_CODES, - _METRIC_BAND_NAMES, - _SHORT_VEG_ENCODED_CODES, - _filter_average_convert_nass_rows, - _nass_yield_dict_from_features, -) -from jdluc.utils.constants import ( - BUSHEL_WEIGHT_KG, - CROP_DRIVER_MAPPING, - EMISSIONS_TYPE_NAMES, - EMISSIVE_ENCODED_CODES, - HA_PER_ACRE, - LAND_USE_CATEGORIES, - NASS_YIELD_YEARS, -) -from jdluc.utils.transitions import encode_transition - - -def _raw_row( - state: str, - commodity: str, - year: int, - bu_per_acre: float, - *, - agg_level: str = 'STATE', - unit_desc: str = 'BU / ACRE', -) -> dict[str, object]: - return { - 'state_fips': state, - 'state_name': {'10': 'DELAWARE', '19': 'IOWA', '20': 'KANSAS'}.get( - state, state - ), - 'commodity_desc': commodity, - 'year': year, - 'value_bu_per_acre': bu_per_acre, - 'agg_level_desc': agg_level, - 'unit_desc': unit_desc, - } - - -def _features() -> list[dict[str, object]]: - """Synthetic raw-row NASS fixture: 3 states × 3 commodities × 4 years.""" - # Per-state × per-commodity arithmetic mean reduces to one of the values - # below. 4-year series per (state, commodity) keeps the mean simple. - _values: dict[tuple[str, str], list[float]] = { - ('10', 'CORN'): [170.0, 180.0, 185.0, 185.0], - ('10', 'SOYBEANS'): [48.0, 50.0, 51.0, 51.0], - ('10', 'WHEAT'): [68.0, 70.0, 71.0, 71.0], - ('19', 'CORN'): [195.0, 200.0, 202.0, 203.0], - ('19', 'SOYBEANS'): [58.0, 60.0, 61.0, 61.0], - ('19', 'WHEAT'): [53.0, 55.0, 56.0, 56.0], - ('20', 'CORN'): [125.0, 130.0, 132.0, 133.0], - ('20', 'SOYBEANS'): [38.0, 40.0, 41.0, 41.0], - ('20', 'WHEAT'): [46.0, 48.0, 49.0, 49.0], - } - rows: list[dict[str, object]] = [] - for (state, commodity), values in _values.items(): - for year, value in zip(NASS_YIELD_YEARS, values, strict=True): - rows.append(_raw_row(state, commodity, year, value)) - return rows - - -def test_key_format() -> None: - result = _nass_yield_dict_from_features(_features()) - for key in result: - state, crop_code = key.split('|') - assert len(state) == 2 and state.isdigit() - assert crop_code.isdigit() - - -def test_wheat_fans_out_to_three_cdl_codes() -> None: - result = _nass_yield_dict_from_features(_features()) - # Delaware wheat has four fixture rows (68, 70, 71, 71); mean = 70.0. - # All three CDL codes share the same yield after fan-out. - de_wheat = {code: result[f'10|{code}'] for code in (22, 23, 24)} - assert de_wheat[22] == de_wheat[23] == de_wheat[24] - assert math.isclose(de_wheat[22]['yield_bu_per_acre'], 70.0) - - -def test_corn_and_soybeans_single_code_each() -> None: - result = _nass_yield_dict_from_features(_features()) - assert '10|1' in result # corn - assert '10|5' in result # soybeans - # No spurious codes for corn / soybeans. - assert '10|2' not in result - assert '10|6' not in result - - -def test_key_cardinality() -> None: - # 3 states × (1 corn + 1 soy + 3 wheat codes) = 15 keys. - result = _nass_yield_dict_from_features(_features()) - assert len(result) == 15 - - -def test_yield_conversion_matches_methodology_example() -> None: - # Corn at 180 bu/acre → 180 × BUSHEL_WEIGHT_KG[CORN] / HA_PER_ACRE ≈ 11300 kg/ha. - # Use a 4-year fixture whose mean is exactly 180 bu/acre so we can assert - # the conversion in both units. - rows = [_raw_row('10', 'CORN', year, 180.0) for year in NASS_YIELD_YEARS] - out = _nass_yield_dict_from_features(rows) - assert math.isclose(out['10|1']['yield_bu_per_acre'], 180.0) - expected_kg = 180.0 * BUSHEL_WEIGHT_KG['CORN'] / HA_PER_ACRE - assert math.isclose(out['10|1']['yield_kg_per_ha'], expected_kg, rel_tol=1e-9) - - -def test_out_of_window_years_are_dropped() -> None: - rows = [ - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0] - 10, 1000.0), - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 180.0), - ] - averaged = _filter_average_convert_nass_rows(rows) - # Mean reflects only the in-window row — the 1000 bu/acre outlier is gone. - assert math.isclose(averaged[('10', 'CORN')]['yield_bu_per_acre'], 180.0) - - -def test_non_state_agg_level_is_dropped() -> None: - rows = [ - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 999.0, agg_level='COUNTY'), - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 180.0), - ] - averaged = _filter_average_convert_nass_rows(rows) - assert math.isclose(averaged[('10', 'CORN')]['yield_bu_per_acre'], 180.0) - - -def test_non_bu_acre_unit_is_dropped() -> None: - rows = [ - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 999.0, unit_desc='LB / ACRE'), - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 180.0), - ] - averaged = _filter_average_convert_nass_rows(rows) - assert math.isclose(averaged[('10', 'CORN')]['yield_bu_per_acre'], 180.0) - - -def test_unknown_commodity_is_dropped_silently() -> None: - rows = [ - _raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 180.0), - _raw_row('10', 'BARLEY', NASS_YIELD_YEARS[0], 80.0), - ] - averaged = _filter_average_convert_nass_rows(rows) - assert ('10', 'BARLEY') not in averaged - assert ('10', 'CORN') in averaged - - -def test_missing_state_commodity_pair_is_absent() -> None: - # Delaware corn only; soybeans / wheat rows for state '10' should not - # appear in the fan-out result. - rows = [_raw_row('10', 'CORN', NASS_YIELD_YEARS[0], 180.0)] - out = _nass_yield_dict_from_features(rows) - assert '10|1' in out - assert '10|5' not in out # soybeans - assert '10|22' not in out # wheat - - -def test_empty_features_yield_empty_dict() -> None: - assert _nass_yield_dict_from_features([]) == {} - - -# --------------------------------------------------------------------------- -# Crops-table precomputed driver-pair encoded codes -# --------------------------------------------------------------------------- -# `_FOREST_ENCODED_CODES` and `_SHORT_VEG_ENCODED_CODES` partition -# `EMISSIVE_ENCODED_CODES` by source family per `CROP_DRIVER_MAPPING`. They're -# derived once at module load — these tests guard against a future -# CROP_DRIVER_MAPPING / EMISSIONS_TYPE_NAMES drift breaking the partition -# silently. - - -def test_metric_band_names_has_canonical_10_entries() -> None: - """Every band the crops reducer sums via `repeat(10)` is in the canonical list.""" - assert len(_METRIC_BAND_NAMES) == 10 - assert _METRIC_BAND_NAMES == ( - 'area_total', - 'area_peatland', - 'allocated_forest', - 'allocated_short_veg', - 'allocated_peatland_conversion', - 'allocated_peatland_occupation', - 'allocated_2005', - 'allocated_2010', - 'allocated_2015', - 'allocated_2020', - ) - - -def _expected_codes_for_driver(target_col: str) -> set[int]: - """Independent re-derivation of forest / short_veg encoded codes.""" - return { - EMISSIVE_ENCODED_CODES[i] - for i in range(len(EMISSIVE_ENCODED_CODES)) - if CROP_DRIVER_MAPPING[EMISSIONS_TYPE_NAMES[i]] == target_col - } - - -def test_forest_encoded_codes_match_crop_driver_mapping() -> None: - """`_FOREST_ENCODED_CODES` = encoded codes for forest+wetland_forest sources.""" - assert set(_FOREST_ENCODED_CODES) == _expected_codes_for_driver('pct_forest') - # Sanity: forest_to_cropland (forest=1, cropland=5) must be present. - forest_code = LAND_USE_CATEGORIES['forest'].code - cropland_code = LAND_USE_CATEGORIES['cropland'].code - assert encode_transition(forest_code, cropland_code) in _FOREST_ENCODED_CODES - - -def test_short_veg_encoded_codes_match_crop_driver_mapping() -> None: - """`_SHORT_VEG_ENCODED_CODES` = encoded codes for short_veg+wetland_short_veg sources.""" - assert set(_SHORT_VEG_ENCODED_CODES) == _expected_codes_for_driver('pct_short_veg') - short_veg_code = LAND_USE_CATEGORIES['short_vegetation'].code - cropland_code = LAND_USE_CATEGORIES['cropland'].code - assert encode_transition(short_veg_code, cropland_code) in _SHORT_VEG_ENCODED_CODES - - -def test_forest_and_short_veg_encoded_codes_partition_emissive_vocab() -> None: - """Together they cover every entry of EMISSIVE_ENCODED_CODES, with no overlap.""" - forest_set = set(_FOREST_ENCODED_CODES) - short_veg_set = set(_SHORT_VEG_ENCODED_CODES) - assert forest_set.isdisjoint(short_veg_set) - assert forest_set | short_veg_set == set(EMISSIVE_ENCODED_CODES) - - -# --------------------------------------------------------------------------- -# compute_region_tables cache-check ordering -# --------------------------------------------------------------------------- -# The build_transitions_table / build_crops_table calls trigger eager -# server-side ``value:compute`` RPCs (NASS yield join, county -# reductions). Probing asset existence first turns a cached region into -# a near-instant skip instead of paying that graph-construction cost. - - -def _fake_fips_band() -> MagicMock: - return MagicMock(name='fips_band') - - -def _fake_region_bbox() -> MagicMock: - return MagicMock(name='region_bbox') - - -def test_compute_region_tables_short_circuits_when_both_cached() -> None: - with ( - patch.object(st, 'asset_exists', return_value=True), - patch.object(st, 'build_transitions_table') as build_t, - patch.object(st, 'build_crops_table') as build_c, - patch.object(st, 'export_table_asset') as export, - ): - result = st.compute_region_tables( - land_use_asset_id='projects/p/assets/land_use_iowa_abc', - emissions_asset_id='projects/p/assets/emissions_iowa_abc', - region='iowa', - version='abc', - fips_band=_fake_fips_band(), - region_bbox=_fake_region_bbox(), - ) - - build_t.assert_not_called() - build_c.assert_not_called() - export.assert_not_called() - assert result['transitions_task'] is None - assert result['crops_task'] is None - - -def test_compute_region_tables_builds_only_missing_side_when_one_cached() -> None: - def _fake_exists(asset_id: str) -> bool: - return 'transitions_' in asset_id - - fake_export_task = MagicMock() - with ( - patch.object(st, 'asset_exists', side_effect=_fake_exists), - patch.object(st, 'build_transitions_table') as build_t, - patch.object(st, 'build_crops_table', return_value=MagicMock()) as build_c, - patch.object(st, 'export_table_asset', return_value=fake_export_task) as export, - patch('jdluc.transform.summary_tables.ee.Image'), - ): - result = st.compute_region_tables( - land_use_asset_id='projects/p/assets/land_use_iowa_abc', - emissions_asset_id='projects/p/assets/emissions_iowa_abc', - region='iowa', - version='abc', - fips_band=_fake_fips_band(), - region_bbox=_fake_region_bbox(), - ) - - build_t.assert_not_called() - build_c.assert_called_once() - export.assert_called_once() - assert result['transitions_task'] is None - assert result['crops_task'] is fake_export_task - - -def test_compute_region_tables_builds_both_when_neither_cached() -> None: - fake_task = MagicMock() - with ( - patch.object(st, 'asset_exists', return_value=False), - patch.object(st, 'build_transitions_table', return_value=MagicMock()) as bt, - patch.object(st, 'build_crops_table', return_value=MagicMock()) as bc, - patch.object(st, 'export_table_asset', return_value=fake_task) as export, - patch('jdluc.transform.summary_tables.ee.Image'), - ): - st.compute_region_tables( - land_use_asset_id='projects/p/assets/land_use_iowa_abc', - emissions_asset_id='projects/p/assets/emissions_iowa_abc', - region='iowa', - version='abc', - fips_band=_fake_fips_band(), - region_bbox=_fake_region_bbox(), - ) - - bt.assert_called_once() - bc.assert_called_once() - assert export.call_count == 2 - - -def test_compute_region_tables_force_rebuilds_even_when_cached() -> None: - fake_task = MagicMock() - with ( - patch.object(st, 'asset_exists', return_value=True) as asset_exists_mock, - patch.object(st, 'build_transitions_table', return_value=MagicMock()) as bt, - patch.object(st, 'build_crops_table', return_value=MagicMock()) as bc, - patch.object(st, 'export_table_asset', return_value=fake_task) as export, - patch('jdluc.transform.summary_tables.ee.Image'), - ): - st.compute_region_tables( - land_use_asset_id='projects/p/assets/land_use_iowa_abc', - emissions_asset_id='projects/p/assets/emissions_iowa_abc', - region='iowa', - version='abc', - fips_band=_fake_fips_band(), - region_bbox=_fake_region_bbox(), - force=True, - ) - - asset_exists_mock.assert_not_called() - bt.assert_called_once() - bc.assert_called_once() - assert export.call_count == 2 diff --git a/jdluc/__tests__/test_transform_summary_tables_integration.py b/jdluc/__tests__/test_transform_summary_tables_integration.py deleted file mode 100644 index 0ba879b..0000000 --- a/jdluc/__tests__/test_transform_summary_tables_integration.py +++ /dev/null @@ -1,473 +0,0 @@ -"""Integration tests for the transitions + crops summary tables (Delaware). - -Asserts schema, cardinality, round-trip consistency against the emissions -raster, non-negativity, row-level allocated <= total invariant, crops -emissions-factor plausibility, pct_* column sums, NASS yield attachment, -and county_fips well-formedness. Requires a live GEE connection and -previously exported Delaware land_use, emissions, transitions, and crops -assets (via cli.py). - -All tests are marked @pytest.mark.integration and excluded from CI by -default (run with: ``pytest -m integration``). -""" - -import math -from typing import Any - -import ee -import pytest - -from jdluc.utils.asset_management import list_assets_matching -from jdluc.utils.constants import ( - BUSHEL_WEIGHT_KG, - CROP_CODE_TO_GROUP, - CROP_GROUPS, - CROPS_TABLE_COLUMNS, - EMISSIONS_TYPE_NAMES, - GCP_PROJECT, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - GLAD_EPOCH_PAIRS, - HA_PER_ACRE, - NASS_TO_CROP_GROUP, - PEATLAND_OCCUPATION_EPOCH_LABEL, - TRANSITIONS_TABLE_COLUMNS, -) -from jdluc.utils.states import get_multi_state_boundary -from jdluc.utils.version import compute_transform_version - -from .conftest import DELAWARE_FIPS - -REGION = 'delaware' - -# Delaware has three counties; their 5-digit STATEFP+COUNTYFP codes. -# Kent=10001, New Castle=10003, Sussex=10005. -_DELAWARE_COUNTY_FIPS: set[str] = {'10001', '10003', '10005'} - -# Round-trip consistency slack: max(absolute floor, relative). -# -# The table side reduces per (county, emissions_type, epoch) and then -# computes ``allocated = Σ_(rows) total × weight[epoch]`` — equivalent -# to ``Σ_epoch ((Σ_pixels luc[epoch]) × weight[epoch])``. The raster side -# materializes ``allocated_luc_emissions_2020`` as a per-pixel band -# ``Σ_epoch (luc[epoch] × weight[epoch])``, which is mathematically -# identical but accumulates float32 add-chain drift across the ~5M -# Delaware pixels: empirically ~0.18% on Delaware (~260 tCO2 on a -# ~143K-tCO2 total). Both numbers are correct for what they represent -# — the table sum is the more numerically accurate scalar, the raster -# band is the per-pixel product downstream consumers actually need. -# We allow up to 0.5% relative drift with a 1 tCO2 floor for tiny -# magnitudes. -_ROUND_TRIP_SLACK_TCO2 = 1.0 -_ROUND_TRIP_SLACK_RELATIVE = 5e-3 - - -def _round_trip_slack(raster_sum: float) -> float: - return max(_ROUND_TRIP_SLACK_TCO2, _ROUND_TRIP_SLACK_RELATIVE * abs(raster_sum)) - - -_REDUCE_MAX_PIXELS = int(1e9) - -# Crop-code -> NASS commodity name (e.g., 1 -> 'CORN'), used to look up the -# expected yield_kg_per_ha / yield_bu_per_acre ratio. -_CROP_CODE_TO_COMMODITY: dict[int, str] = { - code: commodity - for commodity, group in NASS_TO_CROP_GROUP.items() - for code in CROP_GROUPS[group] -} - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope='module') -def gee_init() -> None: - """Initialize GEE once for this test module.""" - from jdluc.utils.gee import initialize_gee - - initialize_gee(GCP_PROJECT) - - -@pytest.fixture(scope='module') -def _current_version(gee_init: None) -> str: - return compute_transform_version() - - -def _resolve_asset_id(prefix: str, version: str) -> str: - expected_suffix = f'{prefix}_{REGION}_{version}' - for asset_id in list_assets_matching(f'{prefix}_{REGION}'): - if asset_id.endswith(expected_suffix): - return asset_id - pytest.skip(f'No {prefix} asset for {REGION} v{version}. Run cli.py first.') - - -@pytest.fixture(scope='module') -def transitions_asset_id(_current_version: str) -> str: - return _resolve_asset_id('transitions', _current_version) - - -@pytest.fixture(scope='module') -def crops_asset_id(_current_version: str) -> str: - return _resolve_asset_id('crops', _current_version) - - -@pytest.fixture(scope='module') -def emissions_asset_id(_current_version: str) -> str: - return _resolve_asset_id('emissions', _current_version) - - -@pytest.fixture(scope='module') -def transitions_features(transitions_asset_id: str) -> list[dict[str, Any]]: - fc = ee.FeatureCollection(transitions_asset_id) - info = fc.getInfo() - return [f['properties'] for f in info['features']] - - -@pytest.fixture(scope='module') -def crops_features(crops_asset_id: str) -> list[dict[str, Any]]: - fc = ee.FeatureCollection(crops_asset_id) - info = fc.getInfo() - return [f['properties'] for f in info['features']] - - -@pytest.fixture(scope='module') -def emissions_asset(emissions_asset_id: str) -> Any: - return ee.Image(emissions_asset_id) - - -@pytest.fixture(scope='module') -def delaware_geometry(gee_init: None) -> Any: - return get_multi_state_boundary([DELAWARE_FIPS]) - - -@pytest.fixture(scope='module') -def raster_allocated_totals( - emissions_asset: Any, delaware_geometry: Any -) -> dict[str, float]: - """Reduce the two `allocated_*_2020` emissions bands over Delaware. - - Returns {'luc': float, 'peatland': float}. Used by the round-trip - consistency tests to validate the `transitions` table's allocated - column sums against the raster ground truth. The reduction pins - crs / crsTransform to the canonical GLAD grid so it matches the - table-side reducer in transform/summary_tables.py exactly — passing - `scale=` without crs/crsTransform here would force an on-the-fly - reprojection and silently diverge from the table by several percent. - """ - reduced = ( - emissions_asset.select( - ['allocated_luc_emissions_2020', 'allocated_peatland_emissions_2020'] - ) - .reduceRegion( - reducer=ee.Reducer.sum(), - geometry=delaware_geometry, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=_REDUCE_MAX_PIXELS, - ) - .getInfo() - ) - return { - 'luc': float(reduced['allocated_luc_emissions_2020']), - 'peatland': float(reduced['allocated_peatland_emissions_2020']), - } - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_transitions_schema_columns( - transitions_features: list[dict[str, Any]], -) -> None: - """Every row has exactly the `TRANSITIONS_TABLE_COLUMNS` keys.""" - assert transitions_features, 'transitions table has no rows' - for row in transitions_features: - assert set(row.keys()) == set( - TRANSITIONS_TABLE_COLUMNS - ), f'Row keys {set(row.keys())} != expected {set(TRANSITIONS_TABLE_COLUMNS)}' - - -@pytest.mark.integration -def test_crops_schema_columns(crops_features: list[dict[str, Any]]) -> None: - """Every row has exactly the `CROPS_TABLE_COLUMNS` keys.""" - assert crops_features, 'crops table has no rows' - for row in crops_features: - assert set(row.keys()) == set( - CROPS_TABLE_COLUMNS - ), f'Row keys {set(row.keys())} != expected {set(CROPS_TABLE_COLUMNS)}' - - -@pytest.mark.integration -def test_transitions_cardinality( - transitions_features: list[dict[str, Any]], -) -> None: - """Row count is within the structural bound. - - Upper bound: 3 counties x (9 LUC + 1 peatland_conversion) x 4 epochs - + 3 counties x 1 peatland_occupation = 123 rows. Lower bound of 20 - is a coarse check that Delaware actually has transitions populated - across multiple (county, epoch, type) combinations. - """ - n_rows = len(transitions_features) - assert 20 <= n_rows <= 123, f'transitions row count out of bounds: {n_rows}' - - -@pytest.mark.integration -def test_transitions_round_trip_luc( - transitions_features: list[dict[str, Any]], - raster_allocated_totals: dict[str, float], -) -> None: - """Sum of allocated LUC rows matches raster `allocated_luc_emissions_2020`.""" - luc_type_names = { - name - for name in EMISSIONS_TYPE_NAMES - if name not in ('peatland_conversion', 'peatland_occupation') - } - table_sum = sum( - float(r['allocated_emissions_2020_tco2']) - for r in transitions_features - if r['emissions_type'] in luc_type_names - ) - raster_sum = raster_allocated_totals['luc'] - slack = _round_trip_slack(raster_sum) - assert abs(table_sum - raster_sum) <= slack, ( - f'LUC round-trip mismatch: table={table_sum}, raster={raster_sum}, ' - f'slack={slack}' - ) - - -@pytest.mark.integration -def test_transitions_round_trip_peatland( - transitions_features: list[dict[str, Any]], - raster_allocated_totals: dict[str, float], -) -> None: - """Sum of allocated peatland rows matches raster `allocated_peatland_emissions_2020`.""" - peatland_type_names = {'peatland_conversion', 'peatland_occupation'} - table_sum = sum( - float(r['allocated_emissions_2020_tco2']) - for r in transitions_features - if r['emissions_type'] in peatland_type_names - ) - raster_sum = raster_allocated_totals['peatland'] - slack = _round_trip_slack(raster_sum) - assert abs(table_sum - raster_sum) <= slack, ( - f'Peatland round-trip mismatch: table={table_sum}, raster={raster_sum}, ' - f'slack={slack}' - ) - - -@pytest.mark.integration -def test_non_negativity_transitions( - transitions_features: list[dict[str, Any]], -) -> None: - """All numeric columns in transitions rows are >= 0.""" - numeric_cols = ( - 'total_area_ha', - 'total_emissions_tco2', - 'allocated_emissions_2020_tco2', - ) - for row in transitions_features: - for col in numeric_cols: - assert row[col] >= 0.0, f'{col}={row[col]} negative in row {row}' - - -@pytest.mark.integration -def test_non_negativity_crops(crops_features: list[dict[str, Any]]) -> None: - """All numeric columns in crops rows are >= 0 (NaN pct columns allowed).""" - non_nan_cols = ( - 'total_production_kg', - 'total_production_bu', - 'total_crop_area_ha', - 'peatland_crop_area_ha', - 'yield_kg_per_ha', - 'yield_bu_per_acre', - 'total_allocated_emissions_tco2', - 'emissions_factor_kgco2e_per_kg', - ) - pct_cols = ( - 'pct_forest', - 'pct_short_veg', - 'pct_peatland_conversion', - 'pct_peatland_occupation', - 'pct_epoch_2005', - 'pct_epoch_2010', - 'pct_epoch_2015', - 'pct_epoch_2020', - ) - for row in crops_features: - for col in non_nan_cols: - value = row[col] - assert ( - value is not None and value >= 0.0 - ), f'{col}={value} negative or null in row {row}' - for col in pct_cols: - value = row[col] - if value is None or (isinstance(value, float) and math.isnan(value)): - continue # legitimate NaN on zero-allocated rows - assert value >= 0.0, f'{col}={value} negative in row {row}' - - -@pytest.mark.integration -def test_allocated_le_total_per_row( - transitions_features: list[dict[str, Any]], -) -> None: - """Every transitions row satisfies allocated <= total + slack. - - peatland_occupation rows carry weight 1.0 and should have - allocated == total (within slack). - """ - for row in transitions_features: - total = float(row['total_emissions_tco2']) - allocated = float(row['allocated_emissions_2020_tco2']) - assert ( - allocated <= total + _ROUND_TRIP_SLACK_TCO2 - ), f'allocated={allocated} > total={total} in row {row}' - if row['emissions_type'] == 'peatland_occupation': - assert abs(allocated - total) <= _ROUND_TRIP_SLACK_TCO2, ( - f'peatland_occupation allocated({allocated}) != total({total}) ' - f'in row {row}' - ) - - -@pytest.mark.integration -def test_crops_ef_plausibility( - crops_features: list[dict[str, Any]], -) -> None: - """Each crop group has at least one row, and every row's EF is in [0, 5]. - - Delaware is a low-deforestation mid-Atlantic state; methodology - expectations for row-crop EFs are "a few kg CO2e / kg at most." - """ - groups_seen: set[str] = set() - for row in crops_features: - group = row['crop_group'] - assert group in ('corn', 'soybeans', 'wheat'), f'Unexpected crop_group: {group}' - groups_seen.add(group) - ef = row['emissions_factor_kgco2e_per_kg'] - # Zero-allocated rows produce EF = 0.0 which is fine. NaN would be - # a bug (shouldn't happen since we drop total_crop_area_ha == 0 - # rows before the EF divide). - assert ef is not None and not ( - isinstance(ef, float) and math.isnan(ef) - ), f'NaN EF in row {row}' - assert 0.0 <= ef <= 5.0, f'EF={ef} outside [0, 5] kgCO2e/kg in row {row}' - assert groups_seen == {'corn', 'soybeans', 'wheat'}, ( - f'Missing crop groups: {{\"corn\", \"soybeans\", \"wheat\"}} ' - f'vs seen {groups_seen}' - ) - - -@pytest.mark.integration -def test_pct_columns_sum_to_one(crops_features: list[dict[str, Any]]) -> None: - """pct_* driver set and pct_epoch_* set each sum to 1.0 per row. - - Skips rows where total_allocated_emissions_tco2 == 0 (pct columns are - NaN by design on those rows). Tolerance is 1.5%, the documented - grouped-reducer drift on small denominators. - """ - driver_cols = ( - 'pct_forest', - 'pct_short_veg', - 'pct_peatland_conversion', - 'pct_peatland_occupation', - ) - epoch_cols = ( - 'pct_epoch_2005', - 'pct_epoch_2010', - 'pct_epoch_2015', - 'pct_epoch_2020', - ) - _PCT_TOLERANCE = 1.5e-2 - for row in crops_features: - total_alloc = float(row['total_allocated_emissions_tco2']) - if total_alloc == 0.0: - continue - driver_sum = sum(float(row[c]) for c in driver_cols) - epoch_sum = sum(float(row[c]) for c in epoch_cols) - assert ( - abs(driver_sum - 1.0) < _PCT_TOLERANCE - ), f'driver pct sum = {driver_sum} in row {row}' - assert ( - abs(epoch_sum - 1.0) < _PCT_TOLERANCE - ), f'epoch pct sum = {epoch_sum} in row {row}' - - -@pytest.mark.integration -def test_crops_yield_attached_and_self_consistent( - crops_features: list[dict[str, Any]], -) -> None: - """Every crops row has positive yields and the unit-conversion is consistent. - - `yield_kg_per_ha / yield_bu_per_acre` must equal - `BUSHEL_WEIGHT_KG[commodity] / HA_PER_ACRE` to within 1e-3. - """ - for row in crops_features: - yield_bu = float(row['yield_bu_per_acre']) - yield_kg = float(row['yield_kg_per_ha']) - assert yield_bu > 0, f'yield_bu_per_acre={yield_bu} non-positive in row {row}' - assert yield_kg > 0, f'yield_kg_per_ha={yield_kg} non-positive in row {row}' - crop_code = int(row['crop_code']) - commodity = _CROP_CODE_TO_COMMODITY[crop_code] - expected_ratio = BUSHEL_WEIGHT_KG[commodity] / HA_PER_ACRE - actual_ratio = yield_kg / yield_bu - assert abs(actual_ratio - expected_ratio) < 1e-3, ( - f'yield ratio mismatch for crop_code={crop_code}: ' - f'actual={actual_ratio}, expected={expected_ratio}' - ) - - -@pytest.mark.integration -def test_delaware_county_fips_well_formed( - transitions_features: list[dict[str, Any]], - crops_features: list[dict[str, Any]], -) -> None: - """All county_fips values are in the Delaware set and well-formed (5 digits).""" - for label, rows in ( - ('transitions', transitions_features), - ('crops', crops_features), - ): - seen_fips = {str(row['county_fips']) for row in rows} - for fips in seen_fips: - assert ( - len(fips) == 5 and fips.isdigit() - ), f'{label}: malformed county_fips {fips!r}' - assert fips.startswith('10'), f'{label}: non-Delaware county_fips {fips!r}' - assert seen_fips.issubset( - _DELAWARE_COUNTY_FIPS - ), f'{label}: unexpected counties {seen_fips - _DELAWARE_COUNTY_FIPS}' - # At least one row per Delaware county should be present. - assert ( - seen_fips == _DELAWARE_COUNTY_FIPS - ), f'{label}: missing counties {_DELAWARE_COUNTY_FIPS - seen_fips}' - - -@pytest.mark.integration -def test_transitions_epoch_labels_are_valid( - transitions_features: list[dict[str, Any]], -) -> None: - """Every row's epoch_transition is either a GLAD epoch pair or '2020'.""" - valid_labels = {f'{a}_{b}' for (a, b) in GLAD_EPOCH_PAIRS} | { - PEATLAND_OCCUPATION_EPOCH_LABEL - } - for row in transitions_features: - assert ( - row['epoch_transition'] in valid_labels - ), f'Unexpected epoch_transition: {row["epoch_transition"]!r}' - - -@pytest.mark.integration -def test_crops_crop_code_to_group_consistent( - crops_features: list[dict[str, Any]], -) -> None: - """Every row's crop_group matches the CDL crop_code mapping.""" - for row in crops_features: - crop_code = int(row['crop_code']) - assert row['crop_group'] == CROP_CODE_TO_GROUP[crop_code], ( - f'crop_group={row["crop_group"]!r} != expected ' - f'{CROP_CODE_TO_GROUP[crop_code]!r} for crop_code={crop_code}' - ) diff --git a/jdluc/__tests__/test_utils.py b/jdluc/__tests__/test_utils.py new file mode 100644 index 0000000..7c4ae43 --- /dev/null +++ b/jdluc/__tests__/test_utils.py @@ -0,0 +1,35 @@ +import concurrent.futures +import time + +from jdluc.utils import ( + threadsafe_cache, +) + + +def test_threadsafe_cache_serial() -> None: + @threadsafe_cache + def now() -> float: + time.sleep(0.2) + return time.monotonic() + + result = now() + assert now() == result + assert now() == result + assert now() == result + assert now() == result + + +def test_threadsafe_cache_parallel() -> None: + @threadsafe_cache + def now() -> float: + time.sleep(0.2) + return time.monotonic() + + results: set[float] = set() + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures: list[concurrent.futures.Future[float]] = [] + for _ in range(25): + futures.append(executor.submit(now)) + for future in concurrent.futures.as_completed(futures): + results.add(future.result()) + assert len(results) == 1 diff --git a/jdluc/__tests__/test_utils_asset_management.py b/jdluc/__tests__/test_utils_asset_management.py deleted file mode 100644 index 7e1bb84..0000000 --- a/jdluc/__tests__/test_utils_asset_management.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Unit tests for ``utils/asset_management.py`` (no GEE credentials).""" - -from unittest.mock import patch - -import ee -import pytest - -from jdluc.utils.asset_management import ( - delete_asset_safely, - list_assets_matching, - parse_version_from_asset_id, -) -from jdluc.utils.constants import GEE_ASSET_ROOT - -# ---------- parse_version_from_asset_id ------------------------------------- - - -@pytest.mark.parametrize( - 'asset_id, expected', - [ - # Clean SHA - (f'{GEE_ASSET_ROOT}/land_use_delaware_a0d76ac0aa12', 'a0d76ac0aa12'), - # Dirty SHA - ( - f'{GEE_ASSET_ROOT}/emissions_delaware_a0d76ac0aa12-dirty-3f2b8c01', - 'a0d76ac0aa12-dirty-3f2b8c01', - ), - # Extract-style, single segment - (f'{GEE_ASSET_ROOT}/harris_agb_conus_v2021', 'v2021'), - # Extract-style with multi-segment version - (f'{GEE_ASSET_ROOT}/nass_yields_v2017_2020', 'v2017_2020'), - # No parseable version suffix — name with only base - (f'{GEE_ASSET_ROOT}/ipcc_climate_zones', None), - # Garbage suffix - (f'{GEE_ASSET_ROOT}/land_use_delaware_notasha', None), - # Short hex (11 chars) does not satisfy the 12-char SHA pattern - (f'{GEE_ASSET_ROOT}/land_use_delaware_abcdef012345_', None), - ], -) -def test_parse_version_from_asset_id(asset_id: str, expected: str | None) -> None: - assert parse_version_from_asset_id(asset_id) == expected - - -# ---------- list_assets_matching -------------------------------------------- - - -def test_list_assets_matching_filters_by_prefix() -> None: - """Only assets whose name starts with the full prefix are returned.""" - assets = [ - {'name': f'{GEE_ASSET_ROOT}/land_use_delaware_abcdef012345'}, - {'name': f'{GEE_ASSET_ROOT}/land_use_iowa_abcdef012345'}, - {'name': f'{GEE_ASSET_ROOT}/emissions_delaware_abcdef012345'}, - ] - - with patch.object( - ee.data, 'listAssets', return_value={'assets': assets} - ) as mock_list: - result = list_assets_matching('land_use_delaware') - - assert result == [f'{GEE_ASSET_ROOT}/land_use_delaware_abcdef012345'] - mock_list.assert_called_once_with({'parent': GEE_ASSET_ROOT}) - - -def test_list_assets_matching_paginates() -> None: - """nextPageToken drives a follow-up call with the token attached.""" - page1 = { - 'assets': [{'name': f'{GEE_ASSET_ROOT}/land_use_delaware_a' * 1}], - 'nextPageToken': 'tok', - } - page2 = { - 'assets': [{'name': f'{GEE_ASSET_ROOT}/land_use_delaware_b'}], - } - - with patch.object(ee.data, 'listAssets', side_effect=[page1, page2]) as mock_list: - result = list_assets_matching('land_use_delaware') - - assert result == [ - f'{GEE_ASSET_ROOT}/land_use_delaware_a', - f'{GEE_ASSET_ROOT}/land_use_delaware_b', - ] - assert mock_list.call_count == 2 - mock_list.assert_any_call({'parent': GEE_ASSET_ROOT}) - mock_list.assert_any_call({'parent': GEE_ASSET_ROOT, 'pageToken': 'tok'}) - - -def test_list_assets_matching_handles_empty_assets_field() -> None: - with patch.object(ee.data, 'listAssets', return_value={}): - assert list_assets_matching('land_use_delaware') == [] - - -# ---------- delete_asset_safely --------------------------------------------- - - -def test_delete_asset_safely_happy_path() -> None: - with patch.object(ee.data, 'deleteAsset') as mock_delete: - ok = delete_asset_safely(f'{GEE_ASSET_ROOT}/land_use_delaware_x') - assert ok is True - mock_delete.assert_called_once_with(f'{GEE_ASSET_ROOT}/land_use_delaware_x') - - -def test_delete_asset_safely_dry_run_skips_deletion() -> None: - with patch.object(ee.data, 'deleteAsset') as mock_delete: - ok = delete_asset_safely(f'{GEE_ASSET_ROOT}/land_use_delaware_x', dry_run=True) - assert ok is True - mock_delete.assert_not_called() - - -def test_delete_asset_safely_returns_false_on_ee_exception() -> None: - with patch.object(ee.data, 'deleteAsset', side_effect=ee.EEException('boom')): - ok = delete_asset_safely(f'{GEE_ASSET_ROOT}/land_use_delaware_x') - assert ok is False diff --git a/jdluc/__tests__/test_utils_constants_extract.py b/jdluc/__tests__/test_utils_constants_extract.py deleted file mode 100644 index ff024e8..0000000 --- a/jdluc/__tests__/test_utils_constants_extract.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Unit tests for extract-side metadata. - -These checks lock in the invariants extract/extract.py relies on: every -non-native dataset carries an expected version and source URL, every -versioned asset ID ends with its expected version, and the NASS -(release_date, version) mapping is mechanical (bumping the release date -without registering the new version blows up at import). -""" - -import pytest - -from jdluc.utils import constants - - -def test_non_native_datasets_covers_six_families() -> None: - assert set(constants.NON_NATIVE_DATASETS) == { - 'harris_agb', - 'huang_bgb', - 'nass_yields', - 'ipcc_climate_zones', - 'gfw_peatlands', - 'county_fips', - } - - -def test_county_fips_is_first_in_orchestrator_order() -> None: - # County-FIPS is the cheapest extract (server-side paint with no - # HTTP / GCS round-trip), so the orchestrator runs it first per the - # "fast-failing mis-configurations surface first" convention. Mostly - # this guards the comment in NON_NATIVE_DATASETS from drifting away - # from the actual list order. - assert constants.NON_NATIVE_DATASETS[0] == 'county_fips' - - -def test_gpm_is_not_in_inventory() -> None: - # We use the GFW Global Peatlands composite (CC BY 4.0) rather than - # GPM 2.0 (CC BY-NC-SA). Guards against a careless revert - # that brings GPM back as a live dependency — license flow-through - # would block commercial use of Cornerstone's outputs. - assert 'global_peatland_map' not in constants.DATASET_INVENTORY - for entry in constants.DATASET_INVENTORY.values(): - assert 'GLOBAL-PEATLAND-DATABASE' not in entry['gee_asset_id'] - - -@pytest.mark.parametrize( - 'dataset_key, asset_id, expected_version', - [ - ('harris_agb', constants.GEE_HARRIS_AGB, constants.EXPECTED_HARRIS_VERSION), - ('huang_bgb', constants.GEE_HUANG_BGB, constants.EXPECTED_HUANG_VERSION), - ('nass_yields', constants.GEE_NASS_YIELDS, constants.EXPECTED_NASS_VERSION), - ( - 'ipcc_climate_zones', - constants.GEE_IPCC_CLIMATE_ZONES, - constants.EXPECTED_IPCC_VERSION, - ), - ( - 'gfw_peatlands', - constants.GEE_GFW_PEATLANDS, - constants.EXPECTED_GFW_PEATLANDS_VERSION, - ), - ( - 'county_fips', - constants.GEE_COUNTY_FIPS_LABEL, - constants.EXPECTED_TIGER_COUNTIES_VERSION, - ), - ], -) -def test_asset_id_carries_expected_version( - dataset_key: str, asset_id: str, expected_version: str -) -> None: - assert asset_id.endswith(f'_{expected_version}') - entry = constants.DATASET_INVENTORY[dataset_key] - assert entry['gee_asset_id'] == asset_id - assert entry['expected_version'] == expected_version - assert entry['native_or_extracted'] == 'extracted' - assert entry['source_url'] - - -def test_native_dataset_entries_have_no_extract_metadata() -> None: - # Native datasets intentionally omit expected_version/source_url — the - # orchestrator iterates NON_NATIVE_DATASETS only, never these. - for key, entry in constants.DATASET_INVENTORY.items(): - if entry['native_or_extracted'] == 'native': - assert 'expected_version' not in entry, key - assert 'source_url' not in entry, key - - -def test_nass_version_derivation_is_mechanical() -> None: - # The registered release date maps to the transform-window-aligned - # version tag, and the derivation helper round-trips. - derived = constants._derive_nass_version_from_release_date( - constants.NASS_QUICKSTATS_RELEASE_DATE - ) - assert derived == constants.EXPECTED_NASS_VERSION - expected_window = ( - f'v{constants.NASS_YIELD_YEARS[0]}_{constants.NASS_YIELD_YEARS[-1]}' - ) - assert derived == expected_window - - -def test_nass_version_derivation_rejects_unregistered_release_date() -> None: - with pytest.raises(ValueError, match='has no registered version'): - constants._derive_nass_version_from_release_date('19990101') - - -def test_nass_url_template_formats_with_release_date() -> None: - # Round-trip the template ↔ release date so a template change that - # drops the {date} placeholder is caught at test time. - url = constants.NASS_QUICKSTATS_URL_TEMPLATE.format( - date=constants.NASS_QUICKSTATS_RELEASE_DATE - ) - assert constants.NASS_QUICKSTATS_RELEASE_DATE in url - assert url.endswith('.txt.gz') - - -def test_harris_source_url_is_arcgis_featureserver() -> None: - # Harris extract uses the ArcGIS FeatureServer query endpoint to - # discover per-tile signed download URLs; the source_url therefore - # points at the FeatureServer rather than a direct tile file. - assert 'arcgis.com' in constants.HARRIS_AGB_ARCGIS_FEATURESERVER - assert constants.HARRIS_AGB_ARCGIS_FEATURESERVER.endswith('/query') - entry = constants.DATASET_INVENTORY['harris_agb'] - assert entry['source_url'] == constants.HARRIS_AGB_ARCGIS_FEATURESERVER - - -def test_gfw_peatlands_source_url_is_gfw_url_template() -> None: - # The peatland source is the GFW Global Peatlands raster tile - # composite. The URL template carries {tile_id} and {api_key} - # placeholders; per-tile URLs are formatted at extract time. The - # DATASET_INVENTORY entry publishes a key-elided form of the same - # template (same posture as Harris AGB's FeatureServer URL). - assert 'data-api.globalforestwatch.org' in constants.GFW_PEATLANDS_URL_TEMPLATE - assert '{tile_id}' in constants.GFW_PEATLANDS_URL_TEMPLATE - assert '{api_key}' in constants.GFW_PEATLANDS_URL_TEMPLATE - assert 'gfw_peatlands' in constants.GFW_PEATLANDS_URL_TEMPLATE - assert 'v20230315' in constants.GFW_PEATLANDS_URL_TEMPLATE - - entry = constants.DATASET_INVENTORY['gfw_peatlands'] - assert 'data-api.globalforestwatch.org' in entry['source_url'] - # Inventory exposes the URL template with the api-key placeholder - # elided to ''; the {tile_id} placeholder remains. - assert '' in entry['source_url'] - assert '{tile_id}' in entry['source_url'] - # The actual API key never leaks into the inventory. - assert constants.GFW_DATA_API_KEY not in entry['source_url'] diff --git a/jdluc/__tests__/test_utils_gee.py b/jdluc/__tests__/test_utils_gee.py deleted file mode 100644 index b122651..0000000 --- a/jdluc/__tests__/test_utils_gee.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Unit tests for ``utils/gee.py`` helpers (no GEE credentials).""" - -from unittest.mock import patch - -import ee -import pytest - -from jdluc.utils.gee import ( - asset_exists, - asset_is_populated, - wait_for_tasks, -) - -# ---------- asset_exists ---------------------------------------------------- - - -def test_asset_exists_true_when_getAsset_succeeds() -> None: - with patch.object(ee.data, 'getAsset', return_value={'name': 'x', 'type': 'IMAGE'}): - assert asset_exists('projects/p/assets/x') is True - - -def test_asset_exists_false_when_getAsset_raises() -> None: - with patch.object(ee.data, 'getAsset', side_effect=ee.EEException('not found')): - assert asset_exists('projects/p/assets/x') is False - - -# ---------- asset_is_populated ---------------------------------------------- - - -def test_asset_is_populated_false_when_asset_absent() -> None: - with patch.object(ee.data, 'getAsset', side_effect=ee.EEException('not found')): - assert asset_is_populated('projects/p/assets/x') is False - - -def test_asset_is_populated_true_for_existing_image() -> None: - """Non-collection assets are always treated as populated when present.""" - with patch.object(ee.data, 'getAsset', return_value={'type': 'IMAGE'}): - assert asset_is_populated('projects/p/assets/x') is True - - -def test_asset_is_populated_true_for_existing_table() -> None: - """Tables (FeatureCollections) are always populated when present.""" - with patch.object(ee.data, 'getAsset', return_value={'type': 'TABLE'}): - assert asset_is_populated('projects/p/assets/x') is True - - -def test_asset_is_populated_false_for_empty_image_collection() -> None: - """An IC with no children is the partial-ingest trap we want to catch.""" - with ( - patch.object(ee.data, 'getAsset', return_value={'type': 'IMAGE_COLLECTION'}), - patch.object(ee.data, 'listAssets', return_value={'assets': []}), - ): - assert asset_is_populated('projects/p/assets/ic') is False - - -def test_asset_is_populated_true_for_image_collection_with_children() -> None: - with ( - patch.object(ee.data, 'getAsset', return_value={'type': 'IMAGE_COLLECTION'}), - patch.object( - ee.data, - 'listAssets', - return_value={'assets': [{'name': 'projects/p/assets/ic/tile_x'}]}, - ), - ): - assert asset_is_populated('projects/p/assets/ic') is True - - -# ---------- wait_for_tasks -------------------------------------------------- - - -def _row(task_id: str, state: str, error_message: str = '') -> dict[str, str]: - return {'id': task_id, 'state': state, 'error_message': error_message} - - -def test_wait_for_tasks_single_completed_returns_empty() -> None: - """One task, COMPLETED on first poll → empty failure dict, no sleep.""" - with ( - patch.object( - ee.data, 'getTaskStatus', return_value=[_row('T1', 'COMPLETED')] - ) as mock_status, - patch('jdluc.utils.gee.time.sleep') as mock_sleep, - ): - failed = wait_for_tasks(['T1'], poll_interval_s=0.01, timeout_s=5.0) - assert failed == {} - assert mock_status.call_count == 1 - mock_sleep.assert_not_called() - - -def test_wait_for_tasks_mixed_completed_and_failed() -> None: - """One COMPLETED + one FAILED in same poll → only failed in result.""" - rows = [ - _row('T1', 'COMPLETED'), - _row('T2', 'FAILED', 'manifest invalid'), - ] - with ( - patch.object(ee.data, 'getTaskStatus', return_value=rows), - patch('jdluc.utils.gee.time.sleep'), - ): - failed = wait_for_tasks( - ['T1', 'T2'], - asset_ids_for_logging={'T1': 'asset/one', 'T2': 'asset/two'}, - poll_interval_s=0.01, - timeout_s=5.0, - ) - assert failed == {'T2': 'manifest invalid'} - - -def test_wait_for_tasks_two_poll_sequence() -> None: - """First poll RUNNING, second poll COMPLETED → returns after second poll.""" - sequence = [ - [_row('T1', 'RUNNING')], - [_row('T1', 'COMPLETED')], - ] - with ( - patch.object(ee.data, 'getTaskStatus', side_effect=sequence) as mock_status, - patch('jdluc.utils.gee.time.sleep') as mock_sleep, - ): - failed = wait_for_tasks(['T1'], poll_interval_s=0.01, timeout_s=5.0) - assert failed == {} - assert mock_status.call_count == 2 - # Sleep once between the two polls. - assert mock_sleep.call_count == 1 - - -def test_wait_for_tasks_timeout_when_perpetually_running() -> None: - """Forever-RUNNING → TimeoutError after the configured budget.""" - - def _always_running(ids: list[str]) -> list[dict[str, str]]: - return [_row(tid, 'RUNNING') for tid in ids] - - fake_now = iter([0.0, 0.0, 100.0, 200.0]) - - def _monotonic() -> float: - try: - return next(fake_now) - except StopIteration: - return 1e9 - - with ( - patch.object(ee.data, 'getTaskStatus', side_effect=_always_running), - patch('jdluc.utils.gee.time.sleep'), - patch('jdluc.utils.gee.time.monotonic', _monotonic), - pytest.raises(TimeoutError), - ): - wait_for_tasks(['T1'], poll_interval_s=0.01, timeout_s=10.0) - - -def test_wait_for_tasks_chunks_at_100() -> None: - """250 tasks → getTaskStatus called with chunk sizes 100, 100, 50.""" - task_ids = [f'T{i}' for i in range(250)] - chunk_sizes: list[int] = [] - - def _capture(ids: list[str]) -> list[dict[str, str]]: - chunk_sizes.append(len(ids)) - return [_row(tid, 'COMPLETED') for tid in ids] - - with ( - patch.object(ee.data, 'getTaskStatus', side_effect=_capture), - patch('jdluc.utils.gee.time.sleep'), - ): - failed = wait_for_tasks(task_ids, poll_interval_s=0.01, timeout_s=5.0) - - assert failed == {} - # Set comparison — wait_for_tasks builds the chunk list from a set, - # so chunk *order* isn't deterministic, but chunk *sizes* are. - assert sorted(chunk_sizes) == [50, 100, 100] diff --git a/jdluc/__tests__/test_utils_transitions.py b/jdluc/__tests__/test_utils_transitions.py deleted file mode 100644 index 42af380..0000000 --- a/jdluc/__tests__/test_utils_transitions.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Unit tests for scalar transition encoding (no GEE dependency). - -Cross-form consistency between ``encode_transition`` and -``encode_transition_image`` is validated by the transform integration -test against a real exported Delaware ``land_use`` asset; no separate -consistency test is needed here. -""" - -import pytest - -from jdluc.utils.transitions import encode_transition - - -def test_no_transition_sentinel() -> None: - """``encode_transition(a, a)`` collapses to the 0 sentinel for every ``a``.""" - for a in range(16): - assert encode_transition(a, a) == 0 - - -def test_known_values() -> None: - """Sanity-check the bit layout against the spec's worked examples.""" - assert encode_transition(1, 5) == 0x15 - assert encode_transition(2, 8) == 0x28 - - -def test_out_of_range_raises() -> None: - """Catches accidental addition of a 17th category.""" - with pytest.raises(AssertionError): - encode_transition(16, 0) - with pytest.raises(AssertionError): - encode_transition(0, 16) - with pytest.raises(AssertionError): - encode_transition(-1, 0) - with pytest.raises(AssertionError): - encode_transition(0, -1) diff --git a/jdluc/__tests__/test_utils_version.py b/jdluc/__tests__/test_utils_version.py deleted file mode 100644 index 662d47a..0000000 --- a/jdluc/__tests__/test_utils_version.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Unit tests for stage-scoped pipeline versioning (no GEE dependency).""" - -import os -import re -import subprocess - -from jdluc.utils.version import ( - compute_publish_version, - compute_transform_version, -) - - -def _head_sha() -> str: - return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() - - -def _repo_root() -> str: - return subprocess.check_output( - ['git', 'rev-parse', '--show-toplevel'], text=True - ).strip() - - -# ---------- format + determinism -------------------------------------------- - - -def test_transform_version_format_clean() -> None: - """Transform version starts with HEAD[:12] and matches the expected regex. - - The working tree may be dirty with respect to ``transform/`` + ``utils/`` - (e.g. these very tests are untracked), in which case the ``-dirty-...`` - suffix is correct. - """ - version = compute_transform_version() - assert version.startswith(_head_sha()[:12]) - assert re.fullmatch(r'[0-9a-f]{12}(-dirty-[0-9a-f]{8})?', version) - - -def test_publish_version_format_clean() -> None: - version = compute_publish_version() - assert version.startswith(_head_sha()[:12]) - assert re.fullmatch(r'[0-9a-f]{12}(-dirty-[0-9a-f]{8})?', version) - - -def test_transform_version_deterministic() -> None: - assert compute_transform_version() == compute_transform_version() - - -def test_publish_version_deterministic() -> None: - assert compute_publish_version() == compute_publish_version() - - -# ---------- stage scoping --------------------------------------------------- - - -def test_transform_version_ignores_publish_files() -> None: - """An untracked file under ``publish/`` does not change the transform SHA.""" - tmp_path = os.path.join(_repo_root(), 'jdluc', 'publish', '_tmp_version_probe') - before = compute_transform_version() - try: - with open(tmp_path, 'w') as f: - f.write('probe\n') - after = compute_transform_version() - assert before == after - finally: - if os.path.exists(tmp_path): - os.remove(tmp_path) - - -def test_publish_version_ignores_transform_files() -> None: - """An untracked file under ``transform/`` does not change the publish SHA.""" - tmp_path = os.path.join(_repo_root(), 'jdluc', 'transform', '_tmp_version_probe') - before = compute_publish_version() - try: - with open(tmp_path, 'w') as f: - f.write('probe\n') - after = compute_publish_version() - assert before == after - finally: - if os.path.exists(tmp_path): - os.remove(tmp_path) - - -def test_utils_change_affects_both_versions() -> None: - """Touching ``utils/`` changes both transform and publish SHAs.""" - tmp_path = os.path.join(_repo_root(), 'jdluc', 'utils', '_tmp_version_probe') - t_before = compute_transform_version() - p_before = compute_publish_version() - try: - with open(tmp_path, 'w') as f: - f.write('probe\n') - t_after = compute_transform_version() - p_after = compute_publish_version() - assert t_before != t_after - assert p_before != p_after - assert '-dirty-' in t_after - assert '-dirty-' in p_after - finally: - if os.path.exists(tmp_path): - os.remove(tmp_path) diff --git a/jdluc/attribution.py b/jdluc/attribution.py new file mode 100644 index 0000000..3b89672 --- /dev/null +++ b/jdluc/attribution.py @@ -0,0 +1,289 @@ +"""Roll up per-pixel emissions to per-(jurisdiction, crop) totals. + +For each jurisdiction (World Bank admin id) and crop, clips the per-pixel emissions zarr +(`emissions.workflow`) to the jurisdiction polygon and masks to the crop's CDL codes — +restricted to GLAD 2020 cropland unless `--skip-glad-crop-filter` is set — then sums crop +area, peatland crop area, peatland-occupation emissions, and total emissions. Returns a +pandas.DataFrame indexed by (admin level, crop, jurisdiction), cached to GCS (`@gcs.cache`). + +Example invocation: + uv run python jdluc/attribution.py --admin-id USA008 +""" + +import argparse +import collections +import collections.abc +import dataclasses +import enum +import logging +import typing + +import pandas +import xarray +from rioxarray.exceptions import NoDataInBounds + +from jdluc import emissions, gcs, geo, harmonize, tiling +from jdluc.datasets import ( + gfw_global_peatlands, + glad_glcluc, + usda_nass_cdl, + worldbank_jurisdictions, +) + +logger = logging.getLogger(__name__) + + +def len_three_str(s: str) -> str: + assert len(s) == 3, f"len({s=:s}) must be three" + return s + + +def iter_tile_cluster_to_iso_a3s( + iso_a3s: collections.abc.Sequence[str], +) -> collections.abc.Generator[tuple[set[str], set[str]]]: + if len(iso_a3s) == 1 and iso_a3s[0] == "USA": + logger.info("Short-circuiting USA -> CONUS") + yield set(tiling.NAME_TO_TILE_SET[tiling.TileSetName.CONUS]), set(iso_a3s) + else: + logger.info("Determining tile_ids which cover each iso_a3") + iso_a3_to_tile_ids = { + iso_a3: worldbank_jurisdictions.get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_a3, + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL.value, + ) + for iso_a3 in iso_a3s + } + + tile_clusters = tiling.get_tile_clusters( + tile_ids={ + tile_id + for tile_ids in iso_a3_to_tile_ids.values() + for tile_id in tile_ids + } + ) + logger.info(f"Found {len(tile_clusters):d} connected clusters of tile_ids") + + for tile_cluster in tile_clusters: + iso_a3_cluster = { + iso_a3 + for iso_a3, tile_ids in iso_a3_to_tile_ids.items() + if tile_cluster.issuperset(tile_ids) + } + assert iso_a3_cluster + logger.info( + f"Cluster covers {len(iso_a3_cluster):d} ISO_A3's with {len(tile_cluster):d} tile_ids" + ) + yield tile_cluster, iso_a3_cluster + + +@enum.unique +class Crop(enum.Enum): + CORN = (usda_nass_cdl.CropClass.CORN,) + SOYBEANS = (usda_nass_cdl.CropClass.SOYBEANS,) + WHEAT = ( + usda_nass_cdl.CropClass.DURUM_WHEAT, + usda_nass_cdl.CropClass.SPRING_WHEAT, + usda_nass_cdl.CropClass.WINTER_WHEAT, + ) + + +@dataclasses.dataclass +class JurisdictionalCropEmission: + admin_id: str + admin_level: str + crop_hectares: float + crop_name: str + jurisdiction_name: str + peatland_crop_hectares: float + peatland_occupation_emissions: float + total_emissions: float + + @classmethod + def from_dset( + cls, + admin_id: str, + admin_level: worldbank_jurisdictions.AdminLevel, + crop: Crop, + dset: xarray.Dataset, + jurisdiction_name: str, + skip_glad_crop_filter: bool, + ) -> typing.Self: + logger.info(f"Populating emissions for {admin_id=:s}/{crop.name=:s}") + + crop_class = dset[usda_nass_cdl.DATASET.fully_qualified_band_name] + if not skip_glad_crop_filter: + glad_class = dset[f"land-class:{max(glad_glcluc.YEARS):d}"] + crop_class = crop_class.where( + glad_class == glad_glcluc.LandClass.CROPLAND.value + ) + crop_mask = crop_class.isin([value.value for value in crop.value]) + emissions_per_hectare = dset["emissions-per-hectare:tco2e-per-ha"] + hectares_per_pixel = dset["hectares-per-pixel:ha"] + is_peatland = dset[gfw_global_peatlands.DATASET.fully_qualified_band_name] + peatland_occupation_per_hectare = dset["peatland-occupation:tco2e-per-ha"] + crop_hectares = hectares_per_pixel.where(crop_mask) + # Compute totals in a batch for a performance gain + totals = ( + xarray.Dataset( + { + "crop_hectares": crop_hectares, + "peatland_crop_hectares": crop_hectares.where(is_peatland == 1), + "peatland_occupation_emissions": ( + peatland_occupation_per_hectare * hectares_per_pixel + ).where(crop_mask), + "total_emissions": ( + emissions_per_hectare * hectares_per_pixel + ).where(crop_mask), + } + ) + .sum() + .compute() + ) + return cls( + admin_id=admin_id, + admin_level=admin_level.name, + crop_hectares=float(totals["crop_hectares"]), + crop_name=crop.name, + jurisdiction_name=jurisdiction_name, + peatland_crop_hectares=float(totals["peatland_crop_hectares"]), + peatland_occupation_emissions=float( + totals["peatland_occupation_emissions"] + ), + total_emissions=float(totals["total_emissions"]), + ) + + @classmethod + def from_constituents( + cls, + admin_id: str, + admin_level: worldbank_jurisdictions.AdminLevel, + constituents: list[JurisdictionalCropEmission], + jurisdiction_name: str, + ) -> typing.Self: + def get_sum(attr: str) -> float: + return sum(getattr(constituent, attr) for constituent in constituents) + + a_constituent = next(iter(constituents)) + return cls( + admin_id=admin_id, + admin_level=admin_level.name, + crop_hectares=get_sum("crop_hectares"), + crop_name=a_constituent.crop_name, + jurisdiction_name=jurisdiction_name, + peatland_crop_hectares=get_sum("peatland_crop_hectares"), + peatland_occupation_emissions=get_sum("peatland_occupation_emissions"), + total_emissions=get_sum("total_emissions"), + ) + + +@gcs.cache(version=1) +def workflow_for_tile_ids( + crops: tuple[Crop, ...], + iso_a3: str, + skip_glad_crop_filter: bool, + tile_ids: tuple[str, ...], +) -> pandas.DataFrame: + logger.info("Calling harmonize and emissions workflows and merging into one dset") + merged = xarray.merge( + [ + harmonize.workflow( + dataset_names=harmonize.DATASET_NAMES, tile_ids=tile_ids + ), + emissions.workflow(tile_ids=tile_ids), + ], + combine_attrs="identical", + compat="identical", + join="exact", + ) + + def it() -> collections.abc.Generator[JurisdictionalCropEmission]: + for ( + admin_id, + admin_name, + geometry, + ) in worldbank_jurisdictions.iter_province_for_iso_a3(iso_a3=iso_a3): + logger.info(f"Clipping to provincial geometry for {admin_id=:s}") + try: + clipped = geo.clip_dset( + dset=merged, + geometry=geometry, + ) + except NoDataInBounds as exc: + logger.warning(repr(exc)) + else: + for crop in crops: + yield JurisdictionalCropEmission.from_dset( + admin_id=str(admin_id), + admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL, + crop=crop, + dset=clipped, + jurisdiction_name=admin_name, + skip_glad_crop_filter=skip_glad_crop_filter, + ) + + provincials = list(it()) + + national_name = str( + worldbank_jurisdictions.get_jurisdiction_for_admin_level( + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL + ).loc[iso_a3]["name"] + ) + logger.info(f"{iso_a3=:s} -> {national_name:s}") + + logger.info("Merging provincial emissions into national emissions") + provincials.append( + JurisdictionalCropEmission.from_constituents( + admin_id=iso_a3, + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL, + constituents=provincials, + jurisdiction_name=national_name, + ) + ) + + return pandas.DataFrame.from_records( + data=map(dataclasses.asdict, provincials) + ).set_index(["admin_level", "crop_name", "jurisdiction_name"]) + + +def workflow( + crops: tuple[Crop, ...], + iso_a3s: collections.abc.Sequence[str], + skip_glad_crop_filter: bool, +) -> pandas.DataFrame: + dfs = ( + workflow_for_tile_ids( + crops=crops, + iso_a3=iso_a3, + skip_glad_crop_filter=skip_glad_crop_filter, + tile_ids=tuple(sorted(tile_cluster)), + ) + for tile_cluster, iso_a3_cluster in iter_tile_cluster_to_iso_a3s( + iso_a3s=iso_a3s + ) + for iso_a3 in iso_a3_cluster + ) + return pandas.concat(list(dfs)).sort_index() + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("iso_a3s", nargs=argparse.ZERO_OR_MORE, type=len_three_str) + parser.add_argument("--skip-glad-crop-filter", action="store_true") + args = parser.parse_args() + + df = workflow( + crops=tuple(Crop), + iso_a3s=args.iso_a3s or ["USA"], + skip_glad_crop_filter=args.skip_glad_crop_filter, + ) + print(df.to_string()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/jdluc/cli.py b/jdluc/cli.py deleted file mode 100644 index 96a2aaf..0000000 --- a/jdluc/cli.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Run the jdLUC pipeline for one region (default: Delaware). - -Usage examples: - uv run python jdluc/cli.py - uv run python jdluc/cli.py --state 10 --region delaware - uv run python jdluc/cli.py --states 19,31,46 --region great_plains_test - uv run python jdluc/cli.py --region great_plains_test - uv run python jdluc/cli.py --region conus - uv run python jdluc/cli.py --force -v - -REGIONS registry below is dev scaffolding for local iteration — named -multi-state regions whose state lists live in one place. Unknown ---region values are treated as free-form labels paired with --state / ---states. -""" - -import argparse -import logging -import sys -from typing import TypedDict - -from jdluc.utils.constants import CONUS_STATE_FIPS, GCP_PROJECT - - -class _RegionSpec(TypedDict): - states: list[str] - region_name: str - - -REGIONS: dict[str, _RegionSpec] = { - # Single state. - 'delaware': {'states': ['10'], 'region_name': 'delaware'}, - # Multi-state test: Corn Belt + northern Plains. - # Iowa=19, Nebraska=31, South Dakota=46. - 'great_plains_test': { - 'states': ['19', '31', '46'], - 'region_name': 'great_plains_test', - }, - # Full 48-state + DC list from utils/constants.py. - 'conus': {'states': list(CONUS_STATE_FIPS), 'region_name': 'conus'}, -} - - -def _resolve_states_and_region(args: argparse.Namespace) -> tuple[list[str], str]: - """Resolve the effective (states, region_name) from CLI args. - - Precedence: - 1. --states (explicit comma-separated FIPS list) + --region. - 2. --region matches REGIONS registry -> lookup. - 3. --state (single) + --region. - """ - if args.states: - states = [fips.strip() for fips in args.states.split(',') if fips.strip()] - return states, args.region - if args.region in REGIONS: - entry = REGIONS[args.region] - return list(entry['states']), entry['region_name'] - return [args.state], args.region - - -def main() -> int: - parser = argparse.ArgumentParser( - description='Run the jdLUC pipeline for one region.' - ) - parser.add_argument( - '--state', - default='10', - help='Single state FIPS code (default: 10 = Delaware). Ignored if --states is set.', - ) - parser.add_argument( - '--states', - default='', - help='Comma-separated state FIPS codes (e.g. 19,31,46). Takes precedence over --state.', - ) - parser.add_argument( - '--region', - default='delaware', - help=( - 'Region label for asset naming. If it matches a REGIONS key ' - '(delaware/great_plains_test/conus) the state list is resolved ' - 'from the registry; otherwise treated as a free-form label ' - 'paired with --state / --states.' - ), - ) - parser.add_argument( - '--gcp-project', - default=GCP_PROJECT, - help=( - 'GCP project ID (default: utils.constants.GCP_PROJECT — edit ' - 'that file to repoint at a different deployment).' - ), - ) - parser.add_argument( - '--force', - action='store_true', - help='Force re-export even if the asset already exists', - ) - parser.add_argument( - '--verbose', - '-v', - action='store_true', - help='Enable verbose (DEBUG) logging', - ) - args = parser.parse_args() - - logging.basicConfig( - level=logging.DEBUG if args.verbose else logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - - states, region_name = _resolve_states_and_region(args) - - from jdluc.pipeline import run_pipeline - - result = run_pipeline( - gcp_project=args.gcp_project, - states=states, - region_name=region_name, - force=args.force, - ) - - status = 'loaded from cache' if result.from_cache else 'computed + exported' - print(f'\nPipeline version: {result.version}') - print(f'Region: {result.region_name}') - print(f'States: {result.states}') - if result.extract_result is not None: - er = result.extract_result - print( - f'\nExtract: cached={er.cached} extracted={er.extracted} ' - f'failed={list(er.failed)}' - ) - print(f'\nland_use asset: {result.land_use_asset_id}') - print(f'emissions asset: {result.emissions_asset_id}') - print(f'transitions table: {result.transitions_table_id}') - print(f'crops table: {result.crops_table_id}') - if result.publish_result is not None: - from jdluc.publish.publish import target_entries - - pr = result.publish_result - print('\nPublish:') - for name, target in target_entries(pr): - kind = 'cached' if target.from_cache else 'exported' - print(f' {name:<11s}: {target} ({kind})') - print(f' publish_version: {pr.transitions.publish_version}') - print(f'\nStatus: {status}') - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/jdluc/config.py b/jdluc/config.py new file mode 100644 index 0000000..cf71c4f --- /dev/null +++ b/jdluc/config.py @@ -0,0 +1,32 @@ +import dataclasses +import functools +import logging +import typing + +import dotenv + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Config: + ingest_bucket_name: str + gcp_project: str + number_of_dask_workers: int + scratch_bucket_name: str + usda_nass_api_key: str + + @classmethod + @functools.cache + def from_dot_env(cls) -> typing.Self: + logger.info("Searching for a .env file") + path_to_env = dotenv.find_dotenv(raise_error_if_not_found=True) + logger.info(f"Loading configs from {path_to_env=:s}") + configs = dotenv.dotenv_values(path_to_env) + types = typing.get_type_hints(cls) + return cls( + **{ + field.name: types[field.name](configs[field.name.upper()]) + for field in dataclasses.fields(cls) + } + ) diff --git a/jdluc/datasets/__init__.py b/jdluc/datasets/__init__.py new file mode 100644 index 0000000..d1f55cb --- /dev/null +++ b/jdluc/datasets/__init__.py @@ -0,0 +1,54 @@ +import enum + +from jdluc.datasets import ( + base, + gfw_global_peatlands, + gfw_harris_agb, + glad_glcluc, + huang_bgb, + ipcc_climate_zones, + soilgrids_ocs, + usda_nass_cdl, + usda_nass_quickstats, + worldbank_jurisdictions, +) + + +class DatasetName(enum.Enum): + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: list[str] + ) -> str: + return name + + GFW_GLOBAL_PEATLANDS = enum.auto() + GFW_HARRIS_AGB = enum.auto() + GLAD_GLCLUC = enum.auto() + HUANG_BGB = enum.auto() + IPCC_CLIMATE_ZONES = enum.auto() + SOILGRIDS_OCS = enum.auto() + USDA_NASS_CDL = enum.auto() + USDA_NASS_QUICKSTATS = enum.auto() + WORLD_BANK_ADMIN_0 = enum.auto() + WORLD_BANK_ADMIN_1 = enum.auto() + WORLD_BANK_ADMIN_2 = enum.auto() + + +NAME_TO_CLS: dict[ + DatasetName, base.RasterDataset | base.TabularDataset | base.VectorDataset +] = { + DatasetName.GFW_GLOBAL_PEATLANDS: gfw_global_peatlands.DATASET, + DatasetName.GFW_HARRIS_AGB: gfw_harris_agb.DATASET, + DatasetName.GLAD_GLCLUC: glad_glcluc.DATASET, + DatasetName.HUANG_BGB: huang_bgb.DATASET, + DatasetName.IPCC_CLIMATE_ZONES: ipcc_climate_zones.DATASET, + DatasetName.SOILGRIDS_OCS: soilgrids_ocs.DATASET, + DatasetName.USDA_NASS_CDL: usda_nass_cdl.DATASET, + DatasetName.USDA_NASS_QUICKSTATS: usda_nass_quickstats.DATASET, + DatasetName.WORLD_BANK_ADMIN_0: worldbank_jurisdictions.ADMIN_0_DATASET, + DatasetName.WORLD_BANK_ADMIN_1: worldbank_jurisdictions.ADMIN_1_DATASET, + DatasetName.WORLD_BANK_ADMIN_2: worldbank_jurisdictions.ADMIN_2_DATASET, +} +assert set(DatasetName) == set(NAME_TO_CLS), ( + f"{set(DatasetName)=:}; {set(NAME_TO_CLS)=:}" +) diff --git a/jdluc/datasets/base.py b/jdluc/datasets/base.py new file mode 100644 index 0000000..82446b7 --- /dev/null +++ b/jdluc/datasets/base.py @@ -0,0 +1,212 @@ +import collections.abc +import dataclasses +import enum +import logging +import os +import tempfile +import typing + +import pandas +import rasterio.enums + +from jdluc import gcs, geo, tiling, utils + +logger = logging.getLogger(__name__) + + +class AssetType(enum.Enum): + @typing.override + def __str__(self) -> str: + return self.name.lower().replace("_", "-") + + RASTER = enum.auto() + VECTOR = enum.auto() + TABULAR = enum.auto() + + +SaveTileIdToLocalPathType = collections.abc.Callable[[str, str], None] + + +@dataclasses.dataclass +class RasterDataset: + band_names: list[str] + no_data: float | int | None + partitioning: tiling.Partitioning + product_name: str + resampling: rasterio.enums.Resampling + save_tile_id_to_local_path: SaveTileIdToLocalPathType + source_name: str + version: str + + def get_gcs_prefix(self, tile_id: str) -> str: + return os.path.join( + str(AssetType.RASTER), + self.source_name, + self.product_name, + self.version, + str(self.partitioning), + f"{tile_id:s}.tif", + ) + + def ingest_a_tile( + self, bucket_name: str, gcp_project: str, overwrite: bool, tile_id: str + ) -> str: + # Stage a geotiff to a tmpdir, clean up and convert to COG, then upload to a templated GCS prefix + gcs_uri = gcs.get_uri_from_bucket_name_prefix( + bucket_name=bucket_name, prefix=self.get_gcs_prefix(tile_id=tile_id) + ) + if overwrite or not gcs.gcs_blob_exists( + gcp_project=gcp_project, remote_path=gcs_uri + ): + with tempfile.TemporaryDirectory() as tmpdir: + path_to_geotiff = os.path.join(tmpdir, "geotiff.tif") + self.save_tile_id_to_local_path(path_to_geotiff, tile_id) + geo.validate_geotiff(path_to_geotiff=path_to_geotiff) + geo.set_band_names_for_geotiff( + band_names=self.band_names, path_to_geotiff=path_to_geotiff + ) + path_to_cog = os.path.join(tmpdir, "cog.tif") + geo.convert_geotiff_to_cog( + metadata={ + "watershed-data-version": self.version, + "watershed-processing-time": utils.get_utc_timestamp(), + "watershed-processing-version": utils.get_git_version(), + "watershed-product-name": self.product_name, + "watershed-remote-url": utils.get_git_remote_url(), + "watershed-source-name": self.source_name, + }, + no_data=self.no_data, + path_to_cog=path_to_cog, + path_to_geotiff=path_to_geotiff, + ) + gcs.upload_local_path_to_gcs( + gcp_project=gcp_project, local_path=path_to_cog, remote_path=gcs_uri + ) + else: + logger.warning( + f"Skipping upload of {tile_id=:s} to GCS because {gcs_uri=:s} " + f"exists and {overwrite=:}" + ) + return gcs_uri + + @property + def fully_qualified_band_names(self) -> list[str]: + return [ + ":".join((self.source_name, self.product_name, band_name)) + for band_name in self.band_names + ] + + @property + def fully_qualified_band_name(self) -> str: + (ret,) = self.fully_qualified_band_names + return ret + + +@dataclasses.dataclass +class VectorDataset: + id_column_names: tuple[str, ...] + name_column_names: tuple[str, ...] + product_name: str + save_tile_id_to_local_path: SaveTileIdToLocalPathType + source_name: str + version: str + + @property + def partitioning(self) -> tiling.Partitioning: + return tiling.Partitioning.WHOLE_WORLD + + def get_gcs_prefix(self, tile_id: str) -> str: + return os.path.join( + str(AssetType.VECTOR), + self.source_name, + self.product_name, + self.version, + str(self.partitioning), + f"{tile_id:s}.fgb", + ) + + def ingest_a_tile( + self, bucket_name: str, gcp_project: str, overwrite: bool, tile_id: str + ) -> str: + # Stage a vector file to a tmpdir, clean up and convert to flatgeobuf, then upload to a templated GCS prefix + gcs_uri = gcs.get_uri_from_bucket_name_prefix( + bucket_name=bucket_name, prefix=self.get_gcs_prefix(tile_id=tile_id) + ) + if overwrite or not gcs.gcs_blob_exists( + gcp_project=gcp_project, remote_path=gcs_uri + ): + with tempfile.TemporaryDirectory() as tmpdir: + path_to_vector = os.path.join(tmpdir, "vector.dat") + self.save_tile_id_to_local_path(path_to_vector, tile_id) + path_to_flatgeobuf = os.path.join(tmpdir, "vector.fgb") + geo.convert_vector_to_flatgeobuf( + id_column_names=self.id_column_names, + name_column_names=self.name_column_names, + path_to_flatgeobuf=path_to_flatgeobuf, + path_to_vector=path_to_vector, + ) + gcs.upload_local_path_to_gcs( + gcp_project=gcp_project, + local_path=path_to_flatgeobuf, + remote_path=gcs_uri, + ) + else: + logger.warning( + f"Skipping upload of {tile_id=:s} to GCS because {gcs_uri=:s} " + f"exists and {overwrite=:}" + ) + return gcs_uri + + +GetRecordsForTileIdType = typing.Callable[[str], list[dict[str, str | float]]] + + +@dataclasses.dataclass +class TabularDataset: + get_records_for_tile_id: GetRecordsForTileIdType + idx_column_names: list[str] + product_name: str + source_name: str + version: str + + @property + def partitioning(self) -> tiling.Partitioning: + return tiling.Partitioning.WHOLE_WORLD + + def get_gcs_prefix(self, tile_id: str) -> str: + return os.path.join( + str(AssetType.TABULAR), + self.source_name, + self.product_name, + self.version, + str(self.partitioning), + f"{tile_id:s}.parquet", + ) + + def ingest_a_tile( + self, bucket_name: str, gcp_project: str, overwrite: bool, tile_id: str + ) -> str: + gcs_uri = gcs.get_uri_from_bucket_name_prefix( + bucket_name=bucket_name, prefix=self.get_gcs_prefix(tile_id=tile_id) + ) + if overwrite or not gcs.gcs_blob_exists( + gcp_project=gcp_project, remote_path=gcs_uri + ): + with tempfile.TemporaryDirectory() as tmpdir: + local_path = os.path.join(tmpdir, "data.parquet") + records = self.get_records_for_tile_id(tile_id) + df = ( + pandas.DataFrame.from_records(data=records) + .set_index(self.idx_column_names) + .sort_index() + ) + df.to_parquet(path=local_path) + gcs.upload_local_path_to_gcs( + gcp_project=gcp_project, local_path=local_path, remote_path=gcs_uri + ) + else: + logger.warning( + f"Skipping upload of {tile_id=:s} to GCS because {gcs_uri=:s} " + f"exists and {overwrite=:}" + ) + return gcs_uri diff --git a/jdluc/datasets/gfw_global_peatlands.py b/jdluc/datasets/gfw_global_peatlands.py new file mode 100644 index 0000000..ca90dbb --- /dev/null +++ b/jdluc/datasets/gfw_global_peatlands.py @@ -0,0 +1,47 @@ +"""Global Forest Watch | Global Peatlands + +license: CC BY 4.0 + +year: ~static + +- Crezee, B. et al. Mapping peat thickness and carbon stocks of the central Congo Basin using field data. Nature Geoscience 15: 639-644 (2022). https://www.nature.com/articles/s41561-022-00966-7. Data downloaded from https://congopeat.net/maps/, using classes 4 and 5 only (peat classes). +- Gumbricht, T. et al. An expert system model for mapping tropical wetlands and peatlands reveals South America as the largest contributor. Global Change Biology 23, 3581–3599 (2017). https://onlinelibrary.wiley.com/doi/full/10.1111/gcb.13689 +- Hastie, A. et al. Risks to carbon storage from land-use change revealed by peat thickness maps of Peru. Nature Geoscience 15: 369-374 (2022). https://www.nature.com/articles/s41561-022-00923-4 +- Miettinen, J., Shi, C. & Liew, S. C. Land cover distribution in the peatlands of Peninsular Malaysia, Sumatra and Borneo in 2015 with changes since 1990. Global Ecological Conservation. 6, 67– 78 (2016). https://www.sciencedirect.com/science/article/pii/S2351989415300470 +- Xu et al. PEATMAP: Refining estimates of global peatland distribution based on a meta-analysis. CATENA 160: 134-140 (2018). https://www.sciencedirect.com/science/article/pii/S0341816217303004 + +https://data.globalforestwatch.org/datasets/gfw::global-peatlands +""" + +import urllib.parse + +import rasterio.enums + +from jdluc import tiling, utils +from jdluc.datasets import base + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + params_dict = { + "grid": "10/40000", + "pixel_meaning": "is", + # This is a public API key + "x-api-key": "2d60cd88-8348-4c0f-a6d5-bd9adb585a8c", + } | {"tile_id": tile_id} + return utils.save_remote_url_to_local_path( + local_path=local_path, + params=urllib.parse.urlencode(params_dict, safe="/"), + remote_url="https://data-api.globalforestwatch.org/dataset/gfw_peatlands/v20230315/download/geotiff", + ) + + +DATASET = base.RasterDataset( + band_names=["is-peatland"], + no_data=(1 << 8) - 1, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="global-peatlands", + resampling=rasterio.enums.Resampling.nearest, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="gfw", + version="v0", +) diff --git a/jdluc/datasets/gfw_harris_agb.py b/jdluc/datasets/gfw_harris_agb.py new file mode 100644 index 0000000..48b39f7 --- /dev/null +++ b/jdluc/datasets/gfw_harris_agb.py @@ -0,0 +1,49 @@ +"""Global Forest Watch | Aboveground Live Woody Biomass Density + +license: CC BY 4.0 + +year: 2000 + +Global maps of twenty-first century forest carbon fluxes. Nature Climate Change, 11, 234–240. DOI 10.1038/s41558-020-00976-6. + +https://data.globalforestwatch.org/datasets/gfw::aboveground-live-woody-biomass-density + +# Methodology + +- Labels from 637k ground plots and 707k LiDAR waveforms along with regional allometric equations +- Random forest regressor +- Engineered bag of landsat spectral indices, SEM, climate aggregates as covariates +""" + +import urllib.parse + +import rasterio.enums + +from jdluc import tiling, utils +from jdluc.datasets import base + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + params_dict = { + "grid": "10/40000", + "pixel_meaning": "Mg_ha-1", + # This is a public API key + "x-api-key": "2d60cd88-8348-4c0f-a6d5-bd9adb585a8c", + } | {"tile_id": tile_id} + return utils.save_remote_url_to_local_path( + local_path=local_path, + params=urllib.parse.urlencode(params_dict, safe="/"), + remote_url="https://data-api.globalforestwatch.org/dataset/whrc_aboveground_woody_biomass_stock_2000/v1.4/download/geotiff", + ) + + +DATASET = base.RasterDataset( + band_names=["aboveground-biomass-mg-per-ha"], + no_data=(1 << 16) - 1, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="harris-agb", + resampling=rasterio.enums.Resampling.bilinear, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="gfw", + version="v0", +) diff --git a/jdluc/datasets/glad_glcluc.py b/jdluc/datasets/glad_glcluc.py new file mode 100644 index 0000000..f0a75a8 --- /dev/null +++ b/jdluc/datasets/glad_glcluc.py @@ -0,0 +1,107 @@ +"""Global Land Analysis and Discovery | Global Land Cover and Land Use Change, 2000-2020 + +license: CC BY 4.0 + +year: 2000, 2005, 2010, 2015, 2020 + +Potapov P., Hansen M.C., Pickens A., Hernandez-Serna A., Tyukavina A., Turubanova S., Zalles V., Li X., Khan A., Stolle F., Harris N., Song X.-P., Baggett A., Kommareddy I., Kommareddy A. (2022) The global 2000-2020 land cover and land use change dataset derived from the Landsat archive: first results. Frontiers in Remote Sensing https://doi.org/10.3389/frsen.2022.856903 + +https://glad.umd.edu/dataset/GLCLUC2020 + +# Methodology + +- Assemblage of five independent models with covariates from [GLAD ARD](https://glad.umd.edu/ard/home): 16-day composites of landsat from 1997 to present; 30m grid with 8 uint16 bands + 1. [Forest](https://doi.org/10.1016/j.rse.2020.112165): labels from GEDI RH95; random forest regressor; separate model for each 1° tile + 2. [Croplands](https://doi.org/10.1038/s43016-021-00429-z): labels manually curated from Google Earth Engine (bootstrapped in three phases); random forest classifier; across 1° tiles + 3. [Urban](https://doi.org/10.1088/1748-9326/ac46ec): labels from OpenStreetMap; U-Net CNN classifier; random 128x128 pixel patches + 4. [Water](https://doi.org/10.1016/j.rse.2020.111792): labels manually curated from Landsat (bootstrapped in five phases); random forest classifier; engineered bag of landsat spectral indices, SEM + 5. Snow: labels manually curated; random forest classifier; separate model for each geographical region +""" + +import enum +import itertools +import logging +import os +import tempfile + +import rasterio +import rasterio.enums + +from jdluc import tiling, utils +from jdluc.datasets import base + +logger = logging.getLogger(__name__) + + +YEARS = (2000, 2005, 2010, 2015, 2020) +BAND_NAMES = [f"{year=:d}" for year in YEARS] + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + year_to_geotiff: dict[int, str] = {} + year_path: str | None = None + for year in YEARS: + year_to_geotiff[year] = year_path = os.path.join(tmpdir, f"{year:d}.tif") + utils.save_remote_url_to_local_path( + local_path=year_path, + params={}, + remote_url=f"https://storage.googleapis.com/earthenginepartners-hansen/GLCLU2000-2020/v2/{year:d}/{tile_id:s}.tif", + ) + assert year_path is not None + with rasterio.open(year_path) as dataset: + profile = dataset.meta.copy() + profile.update(count=len(year_to_geotiff)) + logger.info(f"Combining {YEARS=:} into separate bands for one GeoTIFF") + with rasterio.open(fp=local_path, mode="w", **profile) as dataset: + for idx, (year, year_path) in enumerate( + sorted(year_to_geotiff.items()), start=1 + ): + with rasterio.open(fp=year_path) as source: + dataset.write(source.read(1), idx) + + +DATASET = base.RasterDataset( + band_names=BAND_NAMES, + no_data=(1 << 8) - 1, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="glcluc", + resampling=rasterio.enums.Resampling.nearest, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="glad", + version="v0", +) + + +class LandClass(enum.Enum): + BUILT_UP = enum.auto() + CROPLAND = enum.auto() + FOREST = enum.auto() + GRASSLAND = enum.auto() + OCEAN = enum.auto() + SNOW_ICE = enum.auto() + WATER = enum.auto() + + +def flatten_ranges(*ranges: range) -> list[int]: + return list(itertools.chain.from_iterable(ranges)) + + +LAND_CLASS_TO_VALUES: dict[LandClass, list[int]] = { + # https://storage.googleapis.com/earthenginepartners-hansen/GLCLU2000-2020/legend.xlsx + LandClass.BUILT_UP: [250], + LandClass.CROPLAND: [244], + LandClass.FOREST: (flatten_ranges(range(25, 49), range(125, 149))), + # NB: although the POC considers 0 as bareground, we include it as grassland here + LandClass.GRASSLAND: (flatten_ranges(range(0, 25), range(100, 125))), + LandClass.OCEAN: [254], + LandClass.SNOW_ICE: [241], + LandClass.WATER: flatten_ranges(range(200, 208)), +} +assert set(LandClass) == set(LAND_CLASS_TO_VALUES) +assert all( + set(left).isdisjoint(set(right)) + for left, right in itertools.combinations( + (values for _, values in LAND_CLASS_TO_VALUES.items()), r=2 + ) +) diff --git a/jdluc/datasets/huang_bgb.py b/jdluc/datasets/huang_bgb.py new file mode 100644 index 0000000..7d03d4f --- /dev/null +++ b/jdluc/datasets/huang_bgb.py @@ -0,0 +1,85 @@ +"""Huang et. al. | A global map of root biomass across the world's forests + +license: CC BY 4.0 + +year: ~2010 (the year of the main covariate AGB) + +Huang, Yuanyuan; Ciais, Phillipe; Santoro, Maurizio; Makowski, David; Chave, Jerome; Schepaschenko, Dmitry; et al. (2020). Supporting data and code for A global map of root biomass across the world's forests. figshare. Dataset. https://doi.org/10.6084/m9.figshare.12199637.v1 + +https://www.researchgate.net/publication/354263169_A_global_map_of_root_biomass_across_the_world's_forests + +# Methodology + +- Labels from 10k field measurements across 465 species and 10 biomes +- Random forest regressor +- 47 covariates: shoot biomass, height, soil properties, climate aggregates, water table properties +""" + +import logging +import os +import tempfile +import zipfile + +import numpy +import rasterio.enums +import xarray + +from jdluc import tiling, utils +from jdluc.datasets import base + +logger = logging.getLogger(__name__) + + +# All tiles share one (expensive) download, so cache it +@utils.threadsafe_cache +def _get_dataarray() -> xarray.DataArray: + with tempfile.TemporaryDirectory() as local_dir: + path_to_zip = os.path.join(local_dir, "data.zip") + utils.save_remote_url_to_local_path( + remote_url="https://ndownloader.figshare.com/files/22432460", + params={}, + local_path=path_to_zip, + ) + logger.info(f"Extracting .nc from {path_to_zip=:s}") + path_to_nc: str | None = None + with zipfile.ZipFile(file=path_to_zip) as zf: + for name in zf.namelist(): + if name.endswith("pergridarea_bgb.nc"): + path_to_nc = zf.extract(name, local_dir) + break + assert path_to_nc is not None + logger.info(f"Loading {path_to_nc=:s} into xarray") + ds: xarray.Dataset = xarray.open_dataset(path_to_nc) + return ( + ds.rio.set_spatial_dims(x_dim="LON", y_dim="LAT") + .rio.write_crs("EPSG:4326")["AROOT"] + .fillna(0) + .rio.write_nodata(numpy.nan) + .load() + ) + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + da = _get_dataarray() + max_lat, min_lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) + tiled = ( + da + # Clip to the tile + .sel(LAT=slice(max_lat - 10, max_lat), LON=slice(min_lon, min_lon + 10)) + # We need to flipud since the array is provided with y increasing + .isel(LAT=slice(None, None, -1)) + ) + logger.info(f"Saving xarray to {local_path=:s} for {tile_id=:s}") + tiled.rio.to_raster(local_path, driver="GTiff", recalc_transform=True) + + +DATASET = base.RasterDataset( + band_names=["belowground-biomass-mg-per-ha"], + no_data=None, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="bgb", + resampling=rasterio.enums.Resampling.bilinear, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="huang", + version="v0", +) diff --git a/jdluc/datasets/ipcc_climate_zones.py b/jdluc/datasets/ipcc_climate_zones.py new file mode 100644 index 0000000..267615d --- /dev/null +++ b/jdluc/datasets/ipcc_climate_zones.py @@ -0,0 +1,55 @@ +"""Intergovernmental Panel on Climate Change | Climate Zones + +license: CC BY 4.0 + +year: ~static + +Calvo Buendia, E et al. (2019). 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories. IPCC, Switzerland. https://doi.org/10.5281/zenodo.7303808 + +https://zenodo.org/records/7303808 + +# Methodology + +- decision tree heuristically tuned into 12 classes +- elevation, temperature, precipitation, potential evapotransporation, frost occurrence +""" + +import enum + +import rasterio.enums + +from jdluc import tiling, utils +from jdluc.datasets import base + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + return utils.save_remote_url_to_local_path( + remote_url="https://zenodo.org/records/7303808/files/IPCC_Climate_Zones_ts_3.25.tif?download=1", + params={}, + local_path=local_path, + ) + + +DATASET = base.RasterDataset( + band_names=["climate-zone"], + no_data=(1 << 8) - 1, + partitioning=tiling.Partitioning.WHOLE_WORLD, + product_name="climate-zones", + resampling=rasterio.enums.Resampling.nearest, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="ipcc", + version="v0", +) + + +class Zone(enum.Enum): + TROPICAL_MONTANE = enum.auto() + TROPICAL_WET = enum.auto() + TROPICAL_MOIST = enum.auto() + TROPICAL_DRY = enum.auto() + WARM_TEMPERATE_MOIST = enum.auto() + WARM_TEMPERATE_DRY = enum.auto() + COOL_TEMPERATE_MOIST = enum.auto() + COOL_TEMPERATE_DRY = enum.auto() + BOREAL_MOIST = enum.auto() + BOREAL_DRY = enum.auto() diff --git a/jdluc/datasets/soilgrids_ocs.py b/jdluc/datasets/soilgrids_ocs.py new file mode 100644 index 0000000..2633856 --- /dev/null +++ b/jdluc/datasets/soilgrids_ocs.py @@ -0,0 +1,56 @@ +"""SoilGrids | Organic carbon stocks + +license: CC BY 4.0 + +year: 2000-2010 (the years of covariates) + +Poggio, L., de Sousa, L. M., Batjes, N. H., Heuvelink, G. B. M., Kempen, B., Ribeiro, E., and Rossiter, D.: SoilGrids 2.0: producing soil information for the globe with quantified spatial uncertainty, SOIL, 7, 217–240, https://doi.org/10.5194/soil-7-217-2021, 2021. + +https://gee-community-catalog.org/projects/isric/ + +# Methodology + +- Labels from 196k soil profiles +- Quantile random forest regressors +- Climate, ecology, geology, LULC, SEM, landsat, MODIS, hydrography +""" + +import urllib.parse + +import rasterio.enums + +from jdluc import tiling, utils +from jdluc.datasets import base + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + max_lat, min_lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) + params_tuples = ( + ("COVERAGEID", "ocs_0-30cm_mean"), + ("FORMAT", "image/tiff"), + ("map", "/map/ocs.map"), + ("OUTPUTCRS", "http://www.opengis.net/def/crs/EPSG/0/4326"), + ("REQUEST", "GetCoverage"), + ("SERVICE", "WCS"), + ("SUBSET", f"lat({max_lat - 10:d},{max_lat:d})"), + ("SUBSET", f"long({min_lon:d},{min_lon + 10:d})"), + ("SUBSETTINGCRS", "http://www.opengis.net/def/crs/EPSG/0/4326"), + ("VERSION", "2.0.1"), + ) + return utils.save_remote_url_to_local_path( + local_path=local_path, + params=urllib.parse.urlencode(params_tuples, safe="/:(),"), + remote_url="https://maps.isric.org/mapserv", + ) + + +DATASET = base.RasterDataset( + band_names=["organic-soil-carbon-mg-per-ha"], + no_data=(1 << 15) - 1, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="organic-carbon-stocks", + resampling=rasterio.enums.Resampling.bilinear, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="soilgrids", + version="v0", +) diff --git a/jdluc/datasets/usda_nass_cdl.py b/jdluc/datasets/usda_nass_cdl.py new file mode 100644 index 0000000..ae27318 --- /dev/null +++ b/jdluc/datasets/usda_nass_cdl.py @@ -0,0 +1,251 @@ +"""USDA National Agricultural Statistics Service | Cropland Data Layer + +license: public domain + +year: 2008, ..., 2025 + +United States Department of Agriculture (USDA) National Agricultural Statistics Service (NASS), 20260227, Cropland Data Layer: USDA NASS, USDA NASS Marketing and Information Services Office, Washington, D.C. + +https://www.nass.usda.gov/Research_and_Science/Cropland/SARS1a.php + +# Methodology + +- Labels from "extensive agricultural ground truth" +- Random forest classifier +- Many remote sensing layers: HLSS, HLSL, NED(3DEP) +""" + +import enum +import logging +import os +import tempfile +import zipfile + +import numpy +import rasterio.enums +import rasterio.transform +import xarray + +from jdluc import geo, tiling, utils +from jdluc.datasets import base + +logger = logging.getLogger(__name__) + + +YEAR = 2020 + + +# All tiles share one (expensive) download+reproject, so cache it +@utils.threadsafe_cache +def _get_dataarray() -> xarray.DataArray: + import rioxarray + + with tempfile.TemporaryDirectory() as local_dir: + path_to_zip = os.path.join(local_dir, "data.zip") + utils.save_remote_url_to_local_path( + remote_url=f"https://www.nass.usda.gov/Research_and_Science/Cropland/Release/datasets/{YEAR:d}_30m_cdls.zip", + params={}, + local_path=path_to_zip, + ) + logger.info(f"Extracting .tif from {path_to_zip=:s}") + path_to_geotiff: str | None = None + with zipfile.ZipFile(file=path_to_zip) as zf: + for name in zf.namelist(): + if name.endswith(f"{YEAR:d}_30m_cdls.tif"): + path_to_geotiff = zf.extract(name, local_dir) + break + assert path_to_geotiff is not None + logger.info(f"Loading {path_to_geotiff=:s} into xarray") + darray = rioxarray.open_rasterio( + path_to_geotiff, + chunks=geo.get_chunk_size( + dtypes=[numpy.dtype("uint8")], number_of_dimensions=2 + ), + ) + assert isinstance(darray, xarray.DataArray) + logger.info(f"Reprojecting from {darray.rio.crs.to_epsg():d} to 4326") + return ( + darray.isel(band=0) + .drop_vars("band") + .rio.reproject(dst_crs=4326, resampling=rasterio.enums.Resampling.nearest) + ) + + +def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: + darray = _get_dataarray() + max_lat, min_lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) + tiled = darray.rio.reproject( + dst_crs=4326, + # At the edges, the source data doesn't cover a full 10° tile + transform=rasterio.transform.from_origin( + min_lon, + max_lat, + 10 / tiling.PIXELS_PER_TEN_DEGREE_TILE, + 10 / tiling.PIXELS_PER_TEN_DEGREE_TILE, + ), + shape=(tiling.PIXELS_PER_TEN_DEGREE_TILE, tiling.PIXELS_PER_TEN_DEGREE_TILE), + resampling=DATASET.resampling, + nodata=DATASET.no_data, + ) + logger.info(f"Saving xarray to {local_path=:s} for {tile_id=:s}") + tiled.rio.to_raster(local_path, driver="GTiff", recalc_transform=False) + + +DATASET = base.RasterDataset( + band_names=["crop-class"], + no_data=0, + partitioning=tiling.Partitioning.TEN_DEGREE_TILE, + product_name="cdl", + resampling=rasterio.enums.Resampling.nearest, + save_tile_id_to_local_path=_save_tile_id_to_local_path, + source_name="usda-nass", + # v1: reproject data so that when it doesn't cover a full tile it doesn't get stretched + version="v1", +) + + +@enum.unique +class CropClass(enum.Enum): + # https://www.nass.usda.gov/Research_and_Science/Cropland/sarsfaqs2.php#what.7 + # NO DATA, BACKGROUND 0 + BACKGROUND = 0 + # CROPS 1-60 + CORN = 1 + COTTON = 2 + RICE = 3 + SORGHUM = 4 + SOYBEANS = 5 + SUNFLOWER = 6 + PEANUTS = 10 + TOBACCO = 11 + SWEET_CORN = 12 + POP_OR_ORN_CORN = 13 + MINT = 14 + BARLEY = 21 + DURUM_WHEAT = 22 + SPRING_WHEAT = 23 + WINTER_WHEAT = 24 + OTHER_SMALL_GRAINS = 25 + DBL_CROP_WINWHT_SOYBEANS = 26 + RYE = 27 + OATS = 28 + MILLET = 29 + SPELTZ = 30 + CANOLA = 31 + FLAXSEED = 32 + SAFFLOWER = 33 + RAPE_SEED = 34 + MUSTARD = 35 + ALFALFA = 36 + OTHER_HAY_NON_ALFALFA = 37 + CAMELINA = 38 + BUCKWHEAT = 39 + SUGARBEETS = 41 + DRY_BEANS = 42 + POTATOES = 43 + OTHER_CROPS = 44 + SUGARCANE = 45 + SWEET_POTATOES = 46 + MISC_VEGS_AND_FRUITS = 47 + WATERMELONS = 48 + ONIONS = 49 + CUCUMBERS = 50 + CHICK_PEAS = 51 + LENTILS = 52 + PEAS = 53 + TOMATOES = 54 + CANEBERRIES = 55 + HOPS = 56 + HERBS = 57 + CLOVER_WILDFLOWERS = 58 + SOD_GRASS_SEED = 59 + SWITCHGRASS = 60 + # NON-CROP 61-65 + FALLOW_IDLE_CROPLAND = 61 + PASTURE_GRASS = 62 + FOREST = 63 + SHRUBLAND = 64 + BARREN = 65 + # CROPS 66-80 + CHERRIES = 66 + PEACHES = 67 + APPLES = 68 + GRAPES = 69 + CHRISTMAS_TREES = 70 + OTHER_TREE_CROPS = 71 + CITRUS = 72 + PECANS = 74 + ALMONDS = 75 + WALNUTS = 76 + PEARS = 77 + # OTHER 81-109 + CLOUDS_NO_DATA = 81 + DEVELOPED = 82 + WATER = 83 + WETLANDS = 87 + NONAG_UNDEFINED = 88 + AQUACULTURE = 92 + # NLCD-DERIVED CLASSES 110-195 + OPEN_WATER = 111 + PERENNIAL_ICE_SNOW = 112 + DEVELOPED_OPEN_SPACE = 121 + DEVELOPED_LOW_INTENSITY = 122 + DEVELOPED_MED_INTENSITY = 123 + DEVELOPED_HIGH_INTENSITY = 124 + BARREN_NLCD = 131 + DECIDUOUS_FOREST = 141 + EVERGREEN_FOREST = 142 + MIXED_FOREST = 143 + SHRUBLAND_NLCD = 152 + GRASSLAND_PASTURE = 176 + WOODY_WETLANDS = 190 + HERBACEOUS_WETLANDS = 195 + # CROPS 195-255 + PISTACHIOS = 204 + TRITICALE = 205 + CARROTS = 206 + ASPARAGUS = 207 + GARLIC = 208 + CANTALOUPES = 209 + PRUNES = 210 + OLIVES = 211 + ORANGES = 212 + HONEYDEW_MELONS = 213 + BROCCOLI = 214 + AVOCADOS = 215 + PEPPERS = 216 + POMEGRANATES = 217 + NECTARINES = 218 + GREENS = 219 + PLUMS = 220 + STRAWBERRIES = 221 + SQUASH = 222 + APRICOTS = 223 + VETCH = 224 + DBL_CROP_WINWHT_CORN = 225 + DBL_CROP_OATS_CORN = 226 + LETTUCE = 227 + DBL_CROP_TRITICALE_CORN = 228 + PUMPKINS = 229 + DBL_CROP_LETTUCE_DURUM_WHT = 230 + DBL_CROP_LETTUCE_CANTALOUPE = 231 + DBL_CROP_LETTUCE_COTTON = 232 + DBL_CROP_LETTUCE_BARLEY = 233 + DBL_CROP_DURUM_WHT_SORGHUM = 234 + DBL_CROP_BARLEY_SORGHUM = 235 + DBL_CROP_WINWHT_SORGHUM = 236 + DBL_CROP_BARLEY_CORN = 237 + DBL_CROP_WINWHT_COTTON = 238 + DBL_CROP_SOYBEANS_COTTON = 239 + DBL_CROP_SOYBEANS_OATS = 240 + DBL_CROP_CORN_SOYBEANS = 241 + BLUEBERRIES = 242 + CABBAGE = 243 + CAULIFLOWER = 244 + CELERY = 245 + RADISHES = 246 + TURNIPS = 247 + EGGPLANTS = 248 + GOURDS = 249 + CRANBERRIES = 250 + DBL_CROP_BARLEY_SOYBEANS = 254 diff --git a/jdluc/datasets/usda_nass_quickstats.py b/jdluc/datasets/usda_nass_quickstats.py new file mode 100644 index 0000000..da4a4bb --- /dev/null +++ b/jdluc/datasets/usda_nass_quickstats.py @@ -0,0 +1,155 @@ +import csv +import dataclasses +import io +import logging +import typing +import urllib.parse + +from jdluc import config, utils +from jdluc.datasets import base, worldbank_jurisdictions + +logger = logging.getLogger(__name__) + +HA_PER_ACRE = 0.40468564 +KG_PER_LB = 0.45359237 +# 7 CFR 810 (US Grain Standards Act) +CROP_NAME_TO_LB_PER_BUSHEL = {"CORN": 56, "SOYBEANS": 60, "WHEAT": 60} + +STATE_FIPS_TO_ADMIN_ID = { + 1: "USA001", # Alabama + 2: "USA002", # Alaska + 4: "USA003", # Arizona + 5: "USA004", # Arkansas + 6: "USA005", # California + 8: "USA006", # Colorado + 9: "USA007", # Connecticut + 10: "USA008", # Delaware + 11: "USA009", # District of Columbia + 12: "USA010", # Florida + 13: "USA011", # Georgia + 15: "USA012", # Hawaii + 16: "USA013", # Idaho + 17: "USA014", # Illinois + 18: "USA015", # Indiana + 19: "USA016", # Iowa + 20: "USA017", # Kansas + 21: "USA018", # Kentucky + 22: "USA019", # Louisiana + 23: "USA020", # Maine + 24: "USA021", # Maryland + 25: "USA022", # Massachusetts + 26: "USA023", # Michigan + 27: "USA024", # Minnesota + 28: "USA025", # Mississippi + 29: "USA026", # Missouri + 30: "USA027", # Montana + 31: "USA028", # Nebraska + 32: "USA029", # Nevada + 33: "USA030", # New Hampshire + 34: "USA031", # New Jersey + 35: "USA032", # New Mexico + 36: "USA033", # New York + 37: "USA034", # North Carolina + 38: "USA035", # North Dakota + 39: "USA036", # Ohio + 40: "USA037", # Oklahoma + 41: "USA038", # Oregon + 42: "USA039", # Pennsylvania + 44: "USA040", # Rhode Island + 45: "USA041", # South Carolina + 46: "USA042", # South Dakota + 47: "USA043", # Tennessee + 48: "USA044", # Texas + 49: "USA045", # Utah + 50: "USA046", # Vermont + 51: "USA047", # Virginia + 53: "USA048", # Washington + 54: "USA049", # West Virginia + 55: "USA050", # Wisconsin + 56: "USA051", # Wyoming +} + + +@dataclasses.dataclass +class Yield: + admin_id: str + admin_level: str + crop_name: str + jurisdiction_name: str + year: int + yield_kg_per_ha: float + + @classmethod + def from_dict(cls, d: dict[str, float | str]) -> typing.Self: + crop_name = str(d["commodity_desc"]) + bu_per_acre = float(str(d["Value"]).replace(",", "")) + return cls( + admin_id=STATE_FIPS_TO_ADMIN_ID[int(d["state_fips_code"])], + admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL.name, + crop_name=crop_name, + jurisdiction_name=str(d["state_name"]), + year=int(d["year"]), + yield_kg_per_ha=( + bu_per_acre + * CROP_NAME_TO_LB_PER_BUSHEL[crop_name] + * KG_PER_LB + / HA_PER_ACRE + ), + ) + + +def get_yield_dicts_from_api(api_key: str) -> list[dict[str, float | str]]: + param_tuples = ( + ("key", api_key), + ("source_desc", "SURVEY"), + ("sector_desc", "CROPS"), + ("statisticcat_desc", "YIELD"), + ("agg_level_desc", "STATE"), + ("unit_desc", "BU / ACRE"), + ("freq_desc", "ANNUAL"), + ("reference_period_desc", "YEAR"), + ("class_desc", "ALL CLASSES"), + ("prodn_practice_desc", "ALL PRODUCTION PRACTICES"), + *[("commodity_desc", c) for c in ("CORN", "SOYBEANS", "WHEAT")], + ("format", "CSV"), + ) + with utils.get_requests_session().request( + method="GET", + params=urllib.parse.urlencode(param_tuples), + url="https://quickstats.nass.usda.gov/api/api_GET/", + ) as response: + response.raise_for_status() + body = response.text + + return list(csv.DictReader(io.StringIO(body))) + + +def _get_records_for_tile(tile_id: str) -> list[dict[str, str | float]]: + api_key = config.Config.from_dot_env().usda_nass_api_key + yield_dicts = get_yield_dicts_from_api(api_key=api_key) + logger.info(f"Received {len(yield_dicts):d} raw yields") + + yields = [ + Yield.from_dict(yield_dict) + for yield_dict in yield_dicts + if int(yield_dict["state_fips_code"]) in STATE_FIPS_TO_ADMIN_ID + if yield_dict["Value"] not in {"(D)", "(Z)", "(S)", "(NA)"} + ] + logger.info(f"After filtering: {len(yields):d} raw yields") + + return list(map(dataclasses.asdict, yields)) + + +DATASET = base.TabularDataset( + get_records_for_tile_id=_get_records_for_tile, + idx_column_names=[ + "admin_level", + "admin_id", + "jurisdiction_name", + "crop_name", + "year", + ], + product_name="quickstats", + source_name="usda-nass", + version="2025", +) diff --git a/jdluc/datasets/worldbank_jurisdictions.py b/jdluc/datasets/worldbank_jurisdictions.py new file mode 100644 index 0000000..56e4278 --- /dev/null +++ b/jdluc/datasets/worldbank_jurisdictions.py @@ -0,0 +1,130 @@ +"""World Bank | Official Boundaries + +license: CC BY 4.0 + +year: 2026 + +https://datacatalog.worldbank.org/search/dataset/0038272/world-bank-official-boundaries +""" + +import collections.abc +import enum +import functools +import logging + +import geopandas +import shapely + +from jdluc import config, gcs, tiling, utils +from jdluc.datasets import base + +logger = logging.getLogger(__name__) + + +class AdminLevel(enum.Enum): + NATIONAL = 0 + PROVINCIAL = 1 + DISTRICT = 2 + + +def _save_tile_id_to_local_path_for_admin_level( + admin_level: AdminLevel, +) -> base.SaveTileIdToLocalPathType: + remote_url = ( + "https://datacatalogfiles.worldbank.org/ddh-published/0038272/2/DR0095370/" + "World Bank Official Boundaries (GeoPackage)/World Bank Official Boundaries - " + f"Admin {admin_level.value:d}.gpkg" + ) + + def inner(local_path: str, tile_id: str) -> None: + utils.save_remote_url_to_local_path( + local_path=local_path, + params={}, + remote_url=remote_url, + ) + + return inner + + +ADMIN_0_DATASET = base.VectorDataset( + id_column_names=("ISO_A3",), + name_column_names=("NAM_0",), + product_name="admin-0", + save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + admin_level=AdminLevel.NATIONAL + ), + source_name="world-bank", + version="v0", +) + +ADMIN_1_DATASET = base.VectorDataset( + id_column_names=("ADM1CD_c",), + name_column_names=("NAM_0", "NAM_1"), + product_name="admin-1", + save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + admin_level=AdminLevel.PROVINCIAL + ), + source_name="world-bank", + version="v0", +) + +ADMIN_2_DATASET = base.VectorDataset( + id_column_names=("ADM2CD_c",), + name_column_names=("NAM_0", "NAM_1", "NAM_2"), + product_name="admin-2", + save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + admin_level=AdminLevel.DISTRICT + ), + source_name="world-bank", + version="v0", +) + + +ADMIN_LEVEL_TO_DATASET = { + AdminLevel.NATIONAL: ADMIN_0_DATASET, + AdminLevel.PROVINCIAL: ADMIN_1_DATASET, + AdminLevel.DISTRICT: ADMIN_2_DATASET, +} +assert set(AdminLevel) == set(ADMIN_LEVEL_TO_DATASET) + + +def get_ten_degree_tile_ids_for_admin_id(admin_id: str, admin_level: int) -> list[str]: + dataset = ADMIN_LEVEL_TO_DATASET[AdminLevel(admin_level)] + path_to_fgb = gcs.get_uri_from_bucket_name_prefix( + bucket_name=config.Config.from_dot_env().ingest_bucket_name, + prefix=dataset.get_gcs_prefix(tile_id="world"), + ) + logger.info( + f"Loading {admin_level=:d}'s geometry for {admin_id=:s}'s from {path_to_fgb=:s}" + ) + geometry = ( + geopandas.read_file(filename=path_to_fgb) + .set_index(keys="id") + .loc[admin_id] + .geometry + ) + assert isinstance(geometry, shapely.Polygon | shapely.MultiPolygon) + return sorted(tiling.iter_ten_degree_tile_id_for_geometry(geometry=geometry)) + + +@functools.cache +def get_jurisdiction_for_admin_level( + admin_level: AdminLevel, +) -> geopandas.GeoDataFrame: + logger.info(f"Loading geometry for {admin_level.name=:s}") + dataset = ADMIN_LEVEL_TO_DATASET[admin_level] + return geopandas.read_file( + filename=gcs.get_uri_from_bucket_name_prefix( + bucket_name=config.Config.from_dot_env().ingest_bucket_name, + prefix=dataset.get_gcs_prefix(tile_id="world"), + ) + ).set_index("id") + + +def iter_province_for_iso_a3( + iso_a3: str, +) -> collections.abc.Generator[tuple[str, str, shapely.Geometry]]: + provincial = get_jurisdiction_for_admin_level(admin_level=AdminLevel.PROVINCIAL) + for admin_id, row in sorted(provincial.iterrows()): + if str(admin_id).startswith(iso_a3): + yield str(admin_id), row["name"], row["geometry"] diff --git a/jdluc/emissions.py b/jdluc/emissions.py new file mode 100644 index 0000000..2d51597 --- /dev/null +++ b/jdluc/emissions.py @@ -0,0 +1,448 @@ +"""Compute per-pixel land-conversion emissions from the harmonized dataset. + +Maps GLAD GLCLUC values to land classes, quantifies vegetation- and soil-carbon emissions +across the 2000-2020 epoch transitions, adds ongoing peatland-occupation emissions, applies the +GHGP 20-year linear discount, and scales to per-pixel tCO2e. Returns an xarray.Dataset (written +to zarr), cached to GCS (`@gcs.cache`). + +Example invocation: + uv run python jdluc/emissions.py DELAWARE +""" + +import argparse +import itertools +import logging +import math +import typing + +import numpy +import xarray + +from jdluc import gcs, geo, harmonize, tiling +from jdluc.datasets import ( + gfw_global_peatlands, + gfw_harris_agb, + glad_glcluc, + huang_bgb, + ipcc_climate_zones, + soilgrids_ocs, +) + +logger = logging.getLogger(__name__) + + +def get_land_class(glad_class: xarray.DataArray) -> xarray.DataArray: + # NB: Because the glad class values are disjoint, we can sum vegetation model carbon + # model enum values without worrying about collisions + ret = sum( + glad_class.isin(values) * numpy.uint8(land_class.value) + for land_class, values in glad_glcluc.LAND_CLASS_TO_VALUES.items() + ) + return typing.cast(xarray.DataArray, ret).rename(None) + + +# IPCC 2006, Vol 4, Ch 4, §4.5 (living woody biomass). +CARBON_PER_BIOMASS_LIVE_WOOD: float = 0.47 +# Global forest mean root-to-shoot ratio from Huang et al. (2021), Earth System Science Data 13:4263-4274 +ROOT_TO_SHOOT_RATIO = 0.25 + + +def get_belowground_carbon( + aboveground_biomass: xarray.DataArray, belowground_biomass: xarray.DataArray +) -> xarray.DataArray: + return ( + ( + # Default to BGB when it is provided + belowground_biomass.where(belowground_biomass > 0, other=0) + # Fallback to AGB * R2S when it isn't + + ROOT_TO_SHOOT_RATIO + * aboveground_biomass.where(belowground_biomass == 0, other=0) + ) + * CARBON_PER_BIOMASS_LIVE_WOOD + ).rename("tcarbon-per-ha") + + +# CDM AR-TOOL-12 dead wood and litter factors, expressed as fractions of above-ground biomass +CLIMATE_ZONE_TO_DEAD_ORGANIC_MATTER_PARAMETERS: dict[ + ipcc_climate_zones.Zone, tuple[float, float] +] = { + ipcc_climate_zones.Zone.TROPICAL_WET: (0.06, 0.01), + ipcc_climate_zones.Zone.TROPICAL_MOIST: (0.01, 0.01), + ipcc_climate_zones.Zone.TROPICAL_DRY: (0.02, 0.04), + ipcc_climate_zones.Zone.TROPICAL_MONTANE: (0.07, 0.01), + ipcc_climate_zones.Zone.WARM_TEMPERATE_MOIST: (0.08, 0.04), + ipcc_climate_zones.Zone.WARM_TEMPERATE_DRY: (0.08, 0.04), + ipcc_climate_zones.Zone.COOL_TEMPERATE_MOIST: (0.08, 0.04), + ipcc_climate_zones.Zone.COOL_TEMPERATE_DRY: (0.08, 0.04), + ipcc_climate_zones.Zone.BOREAL_MOIST: (0.08, 0.04), + ipcc_climate_zones.Zone.BOREAL_DRY: (0.08, 0.04), +} +assert set(ipcc_climate_zones.Zone) == set( + CLIMATE_ZONE_TO_DEAD_ORGANIC_MATTER_PARAMETERS +) + +# CDM AR-TOOL-12 (dead wood and litter pools). +CARBON_PER_BIOMASS_DEAD_WOOD = 0.50 +CARBON_PER_BIOMASS_LITTER = 0.37 + + +def get_dead_organic_matter_carbon( + aboveground_biomass: xarray.DataArray, climate_zones: xarray.DataArray +) -> xarray.DataArray: + ret = sum( + aboveground_biomass.where(climate_zones == climate_zone.value, other=0) + * ( + CARBON_PER_BIOMASS_DEAD_WOOD * dead_wood_fraction + + CARBON_PER_BIOMASS_LITTER * litter_fraction + ) + for climate_zone, ( + dead_wood_fraction, + litter_fraction, + ) in CLIMATE_ZONE_TO_DEAD_ORGANIC_MATTER_PARAMETERS.items() + ) + return typing.cast(xarray.DataArray, ret).rename("tcarbon-per-ha") + + +# Houghton/BLUE total vegetation carbon density for grassland / shrubland pixels +CLIMATE_ZONE_TO_GRASSLAND_TCARBON_PER_HA: dict[ipcc_climate_zones.Zone, float] = { + ipcc_climate_zones.Zone.TROPICAL_WET: 18.0, + ipcc_climate_zones.Zone.TROPICAL_MOIST: 18.0, + ipcc_climate_zones.Zone.TROPICAL_DRY: 7.0, + ipcc_climate_zones.Zone.TROPICAL_MONTANE: 7.0, + ipcc_climate_zones.Zone.WARM_TEMPERATE_MOIST: 7.0, + ipcc_climate_zones.Zone.WARM_TEMPERATE_DRY: 5.0, + ipcc_climate_zones.Zone.COOL_TEMPERATE_MOIST: 7.0, + ipcc_climate_zones.Zone.COOL_TEMPERATE_DRY: 5.0, + ipcc_climate_zones.Zone.BOREAL_MOIST: 6.0, + ipcc_climate_zones.Zone.BOREAL_DRY: 3.0, +} +assert set(ipcc_climate_zones.Zone) == set(CLIMATE_ZONE_TO_GRASSLAND_TCARBON_PER_HA) + + +def get_grassland_carbon(climate_zones: xarray.DataArray) -> xarray.DataArray: + footprint = xarray.ones_like(climate_zones) + ret = sum( + tcarbon_per_ha * footprint.where(climate_zones == climate_zone.value, other=0) + for climate_zone, tcarbon_per_ha in CLIMATE_ZONE_TO_GRASSLAND_TCARBON_PER_HA.items() + ) + return typing.cast(xarray.DataArray, ret).rename("tcarbon-per-ha") + + +def get_vegetation_carbon( + aboveground_carbon: xarray.DataArray, + belowground_carbon: xarray.DataArray, + dead_organic_carbon: xarray.DataArray, + grassland_carbon: xarray.DataArray, + land_class: xarray.DataArray, +) -> xarray.DataArray: + # vegetation = forest + grasslands (all other GLAD classes contain zero carbon) + forest_carbon = aboveground_carbon + belowground_carbon + dead_organic_carbon + return ( + forest_carbon.where(land_class == glad_glcluc.LandClass.FOREST.value, other=0) + + grassland_carbon.where( + land_class == glad_glcluc.LandClass.GRASSLAND.value, other=0 + ) + ).rename("tcarbon-per-ha") + + +CO2E_PER_CARBON = 44 / 12 + + +def get_vegetation_emissions( + after: xarray.DataArray, before: xarray.DataArray +) -> xarray.DataArray: + return ((before - after).clip(min=0) * CO2E_PER_CARBON).rename("tco2e-per-ha") + + +# IPCC 2019 Vol 4 Table 5.5 +CLIMATE_ZONE_TO_SOC_LOSS_FRACTION: dict[ipcc_climate_zones.Zone, float] = { + ipcc_climate_zones.Zone.TROPICAL_WET: 0.83, + ipcc_climate_zones.Zone.TROPICAL_MOIST: 0.83, + ipcc_climate_zones.Zone.TROPICAL_DRY: 0.92, + # Tropical montane factors are approximated as the mean of WARM_TEMPERATE_MOIST and TROPICAL_MOIST_WET + ipcc_climate_zones.Zone.TROPICAL_MONTANE: 0.76, + ipcc_climate_zones.Zone.WARM_TEMPERATE_MOIST: 0.69, + ipcc_climate_zones.Zone.WARM_TEMPERATE_DRY: 0.76, + # Cool Temperate and Boreal share rows in Table 5.5. + ipcc_climate_zones.Zone.COOL_TEMPERATE_MOIST: 0.70, + ipcc_climate_zones.Zone.COOL_TEMPERATE_DRY: 0.77, + ipcc_climate_zones.Zone.BOREAL_MOIST: 0.70, + ipcc_climate_zones.Zone.BOREAL_DRY: 0.77, +} + + +def get_mineral_soil_emissions( + climate_zones: xarray.DataArray, + soil_organic_carbon: xarray.DataArray, +) -> xarray.DataArray: + ret = sum( + soil_organic_carbon.where(climate_zones == climate_zone.value, other=0) + * (1 - soc_loss_fraction) + * CO2E_PER_CARBON + for climate_zone, soc_loss_fraction in CLIMATE_ZONE_TO_SOC_LOSS_FRACTION.items() + ) + return typing.cast(xarray.DataArray, ret).rename("tco2e-per-ha") + + +PEATLAND_EMISSIONS_PULSE_TCO2E_PER_HA = 621 + + +def get_soil_emissions( + after: xarray.DataArray, + before: xarray.DataArray, + climate_zones: xarray.DataArray, + is_peatland: xarray.DataArray, + soil_organic_carbon: xarray.DataArray, +) -> xarray.DataArray: + is_emissive = before.isin( + ( + glad_glcluc.LandClass.FOREST.value, + glad_glcluc.LandClass.GRASSLAND.value, + ) + ) & ( + ~after.isin( + ( + glad_glcluc.LandClass.FOREST.value, + glad_glcluc.LandClass.GRASSLAND.value, + ) + ) + ) + + peatlands_conversion_emissions = ( + PEATLAND_EMISSIONS_PULSE_TCO2E_PER_HA * xarray.ones_like(soil_organic_carbon) + ) + mineral_soil_organic_emissions = get_mineral_soil_emissions( + climate_zones=climate_zones, + soil_organic_carbon=soil_organic_carbon, + ) + + return ( + # is_emissive & is_peatland: peatland pulse + peatlands_conversion_emissions.where(is_emissive & (is_peatland == 1), other=0) + # is_emissive & ~is_peatland: mineral SOC + + mineral_soil_organic_emissions.where( + is_emissive & (is_peatland == 0), other=0 + ) + # ~is_emissive (else): zero + ).rename("tco2e-per-ha") + + +PEATLAND_EMISSIONS_ANNUAL_TCO2E_PER_HA = 37.3 + + +def get_peatland_occupation_emissions( + is_peatland: xarray.DataArray, year_to_land_class: dict[int, xarray.DataArray] +) -> xarray.DataArray: + latest_land_class = year_to_land_class[max(year_to_land_class)] + undrained_classes = ( + # still natural + glad_glcluc.LandClass.FOREST.value, + glad_glcluc.LandClass.GRASSLAND.value, + # still wet + glad_glcluc.LandClass.OCEAN.value, + glad_glcluc.LandClass.SNOW_ICE.value, + glad_glcluc.LandClass.WATER.value, + ) + return PEATLAND_EMISSIONS_ANNUAL_TCO2E_PER_HA * is_peatland.where( + ~latest_land_class.isin(undrained_classes), other=0 + ).rename("tco2e-per-ha") + + +EpochType = tuple[int, int] +EPOCH_TO_LINEAR_DISCOUNT_WEIGHT: dict[EpochType, float] = { + (2000, 2005): 0.0125, + (2005, 2010): 0.0375, + (2010, 2015): 0.0625, + (2015, 2020): 0.0875, +} +assert all((after - before) == 5 for (before, after) in EPOCH_TO_LINEAR_DISCOUNT_WEIGHT) +assert math.isclose(sum(EPOCH_TO_LINEAR_DISCOUNT_WEIGHT.values()), 0.2) + + +def get_linear_discounted_emissions( + epoch_to_emissions: dict[EpochType, xarray.DataArray], +) -> xarray.DataArray: + ret = sum( + emissions * EPOCH_TO_LINEAR_DISCOUNT_WEIGHT[epoch] + for epoch, emissions in epoch_to_emissions.items() + ) + return typing.cast(xarray.DataArray, ret).rename("tco2e-per-ha") + + +def get_hectares_per_pixel(darray: xarray.DataArray) -> xarray.DataArray: + # https://en.wikipedia.org/wiki/Earth%27s_circumference + equator_meters_per_longitude_degrees = 40_075_017 / 360 + meters_per_latitude_degrees = 40_007_863 / 360 + + # NB: coords are 1-D and tiny, so reducing them to scalar spacings is cheap/greedy + delta_longitude_degrees = abs(float(darray.x.diff("x").mean())) + delta_latitude_degrees = abs(float(darray.y.diff("y").mean())) + + return ( + # hectares at equator + (delta_latitude_degrees * meters_per_latitude_degrees) + * (delta_longitude_degrees * equator_meters_per_longitude_degrees) + / 10_000 + # projection to latitude + * numpy.cos(darray.y * numpy.pi / 180) + # broadcast across x while inheriting darray's aligned 2-D chunking + * xarray.ones_like(darray) + ).rename("ha") + + +def get_dset_for_output(name_to_darray: dict[str, xarray.DataArray]) -> xarray.Dataset: + chunk_size = geo.get_chunk_size( + dtypes=[numpy.dtype("float32")] * len(name_to_darray), number_of_dimensions=2 + ) + + def merge_name_units(name: str, units: typing.Hashable | None) -> str: + prefix, _, suffix = name.partition(":") + words = filter(bool, (prefix, units, suffix)) + return ":".join(map(str, words)) + + return xarray.Dataset( + { + merge_name_units(name=name, units=darray.name): geo.unify_dtype_and_no_data( + darray=darray + ).chunk(chunks=chunk_size) + for name, darray in name_to_darray.items() + } + ) + + +@gcs.cache(version=2) +def workflow(tile_ids: tiling.TileSetType) -> xarray.Dataset: + logger.info(f"Running the land conversion and emissions worflow for {tile_ids=:}") + dset = harmonize.workflow(dataset_names=harmonize.DATASET_NAMES, tile_ids=tile_ids) + + logger.info("Mapping GLCLUC values to land classes") + year_to_land_class = { + year: get_land_class(glad_class=dset[band_name]) + for year, band_name in zip( + glad_glcluc.YEARS, + glad_glcluc.DATASET.fully_qualified_band_names, + strict=True, + ) + } + + logger.info("Quantifying vegetation carbon stock and emissions") + aboveground_biomass = dset[gfw_harris_agb.DATASET.fully_qualified_band_name] + aboveground_carbon = (aboveground_biomass * CARBON_PER_BIOMASS_LIVE_WOOD).rename( + "tcarbon-per-ha" + ) + belowground_carbon = get_belowground_carbon( + aboveground_biomass=aboveground_biomass, + belowground_biomass=dset[huang_bgb.DATASET.fully_qualified_band_name], + ) + climate_zones = dset[ipcc_climate_zones.DATASET.fully_qualified_band_name] + dead_organic_matter_carbon = get_dead_organic_matter_carbon( + aboveground_biomass=aboveground_biomass, climate_zones=climate_zones + ) + grassland_carbon = get_grassland_carbon(climate_zones=climate_zones) + year_to_vegetation_carbon = { + year: get_vegetation_carbon( + aboveground_carbon=aboveground_carbon, + belowground_carbon=belowground_carbon, + dead_organic_carbon=dead_organic_matter_carbon, + grassland_carbon=grassland_carbon, + land_class=land_class, + ) + for year, land_class in year_to_land_class.items() + } + epoch_to_vegetation_emissions = { + (before, after): get_vegetation_emissions( + after=year_to_vegetation_carbon[after], + before=year_to_vegetation_carbon[before], + ) + for (before, after) in itertools.pairwise(sorted(year_to_land_class)) + } + + logger.info("Quantifying soil emissions") + is_peatland = dset[gfw_global_peatlands.DATASET.fully_qualified_band_name] + epoch_to_soil_emissions = { + (before, after): get_soil_emissions( + after=year_to_land_class[after], + before=year_to_land_class[before], + climate_zones=climate_zones, + is_peatland=is_peatland, + soil_organic_carbon=dset[soilgrids_ocs.DATASET.fully_qualified_band_name], + ) + for (before, after) in itertools.pairwise(sorted(year_to_land_class)) + } + + logger.info("Summing emissions from vegetation and soil") + epoch_to_emissions: dict[EpochType, xarray.DataArray] = { + epoch: ( + epoch_to_vegetation_emissions[epoch] + epoch_to_soil_emissions[epoch] + ).rename("tco2e-per-ha") + for epoch in epoch_to_vegetation_emissions + } + + logger.info("Quantifying and adding peatland occupation emissions") + peatland_occupation_emissions: xarray.DataArray = get_peatland_occupation_emissions( + is_peatland=is_peatland, year_to_land_class=year_to_land_class + ) + emissions_per_hectare: xarray.DataArray = ( + get_linear_discounted_emissions(epoch_to_emissions=epoch_to_emissions) + + peatland_occupation_emissions + ) + + logger.info("Scaling by area") + hectares_per_pixel = get_hectares_per_pixel(darray=emissions_per_hectare) + emissions = (emissions_per_hectare * hectares_per_pixel).rename("tco2e") + + return get_dset_for_output( + name_to_darray={ + f"land-class:{year:d}": darray + for year, darray in year_to_land_class.items() + } + | { + "aboveground-carbon": aboveground_carbon, + "belowground-carbon": belowground_carbon, + "dead-organic-matter-carbon": dead_organic_matter_carbon, + "grassland-carbon": grassland_carbon, + } + | { + f"vegetation-carbon:{year:d}": darray + for year, darray in year_to_vegetation_carbon.items() + } + | { + f"vegetation-emissions:{before:d}-{after:d}": darray + for (before, after), darray in epoch_to_vegetation_emissions.items() + } + | { + f"soil-emissions:{before:d}-{after:d}": darray + for (before, after), darray in epoch_to_soil_emissions.items() + } + | { + f"emissions:{before:d}-{after:d}": darray + for (before, after), darray in epoch_to_emissions.items() + } + | { + "peatland-occupation": peatland_occupation_emissions, + "emissions-per-hectare": emissions_per_hectare, + "hectares-per-pixel": hectares_per_pixel, + "emissions": emissions, + } + ) + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "tile_set_name", choices=sorted(e.name for e in tiling.TileSetName) + ) + args = parser.parse_args() + + tile_set_name = tiling.TileSetName[str(args.tile_set_name)] + workflow(tile_ids=tiling.NAME_TO_TILE_SET[tile_set_name]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/jdluc/emissions_factors.py b/jdluc/emissions_factors.py new file mode 100644 index 0000000..7456dc1 --- /dev/null +++ b/jdluc/emissions_factors.py @@ -0,0 +1,111 @@ +"""Build the per-(jurisdiction, crop) emissions-factor table. + +Joins the per-(admin, crop) attribution rollup (`attribution.workflow`) to NASS QuickStats +yields (4-year mean) and derives each crop's total production, emissions factor (kgCO2e per +kg), and peatland-occupation fraction. Returns a pandas.DataFrame indexed by (admin level, +crop, jurisdiction), cached to GCS (`@gcs.cache`). + +Example invocation: + uv run python jdluc/emissions_factors.py --admin-id USA008 +""" + +import argparse +import logging + +import pandas + +from jdluc import attribution, config, gcs +from jdluc.datasets import usda_nass_quickstats + +logger = logging.getLogger(__name__) + + +NASS_YIELD_YEARS = (2017, 2018, 2019, 2020) +KG_PER_TONNE = 1000 + + +def load_yields() -> pandas.DataFrame: + gcs_uri = gcs.get_uri_from_bucket_name_prefix( + bucket_name=config.Config.from_dot_env().ingest_bucket_name, + prefix=usda_nass_quickstats.DATASET.get_gcs_prefix(tile_id="world"), + ) + logger.info(f"Loading yields from {gcs_uri=:s}") + return pandas.read_parquet(path=gcs_uri) + + +def merge_emissions_and_yields( + emissions: pandas.DataFrame, + raw_yields: pandas.DataFrame, +) -> pandas.DataFrame: + reduced_yields = ( + raw_yields[raw_yields.index.get_level_values("year").isin(NASS_YIELD_YEARS)] + .groupby(level=["admin_id", "crop_name"])["yield_kg_per_ha"] + .mean() + .reset_index() + ) + + flat = emissions.reset_index() + merged = flat.merge(reduced_yields, how="left", on=["admin_id", "crop_name"]) + unmatched = int(merged["yield_kg_per_ha"].isna().sum()) + if unmatched: + logger.warning(f"{unmatched:d} (admin, crop) row(s) had no matching NASS yield") + + production_kg = merged["crop_hectares"] * merged["yield_kg_per_ha"] + merged["total_production_kg"] = production_kg + merged["emissions_factor_kgco2e_per_kg"] = ( + (merged["total_emissions"] * KG_PER_TONNE) + .div(production_kg) + .where(production_kg > 0) + ) + + merged["peatland_occupation_fraction"] = ( + merged["peatland_occupation_emissions"] + .div(merged["total_emissions"]) + .where(merged["total_emissions"] > 0) + ) + + return merged.set_index( + ["admin_level", "crop_name", "jurisdiction_name"] + ).sort_index() + + +@gcs.cache(version=1) +def workflow( + crops: tuple[attribution.Crop, ...], + iso_a3s: tuple[str, ...], + skip_glad_crop_filter: bool, +) -> pandas.DataFrame: + return merge_emissions_and_yields( + emissions=attribution.workflow( + crops=crops, + iso_a3s=iso_a3s, + skip_glad_crop_filter=skip_glad_crop_filter, + ), + raw_yields=load_yields(), + ) + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "iso_a3s", nargs=argparse.ZERO_OR_MORE, type=attribution.len_three_str + ) + parser.add_argument("--skip-glad-crop-filter", action="store_true") + args = parser.parse_args() + + df = workflow( + crops=tuple(attribution.Crop), + iso_a3s=tuple(sorted(args.iso_a3s or ["USA"])), + skip_glad_crop_filter=args.skip_glad_crop_filter, + ) + print(df.to_string()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/jdluc/extract/__init__.py b/jdluc/extract/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jdluc/extract/_tile_pipeline.py b/jdluc/extract/_tile_pipeline.py deleted file mode 100644 index 05e2714..0000000 --- a/jdluc/extract/_tile_pipeline.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Shared tile-collection orchestrator for tile-based extractors. - -`harris_agb.py` and `gfw_peatlands.py` both ingest a 10°×10° grid of -GeoTIFF tiles from an HTTP source into a GEE ImageCollection. The two -modules used to share ~85% of their structure verbatim — Phase A -concurrent staging via ``ThreadPoolExecutor``, Phase B sequential -ingestion submission, Phase C bulk poll, Phase D finally-block cleanup. -This module owns the orchestrator; per-dataset modules supply the -varying bits (asset paths, band name, URL resolver) and a one-line call. - -The orchestrator's failure model is unchanged from the prior copies: - -- A failed Phase A staging future is logged and the tile is dropped from - Phase B (sibling tiles still go). -- A Phase B submit exception lands the staged blob in ``orphan_blobs`` - so Phase D still cleans GCS. -- A FAILED Phase C task is reported in the warning log; the asset ID is - returned regardless. Re-run with ``force=True`` to retry. -""" - -import logging -import os -import shutil -import tempfile -import uuid -from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor, as_completed - -from jdluc.extract.mirror import fetch_with_mirror -from jdluc.utils.gee import ( - asset_exists, - create_image_collection_if_absent, - delete_asset_if_present, - delete_gcs_blob, - start_ingestion_no_wait, - upload_to_gcs, - wait_for_tasks, -) - -logger = logging.getLogger(__name__) - - -def _tile_image_asset_id(asset_root: str, tile_id: str) -> str: - return f'{asset_root}/tile_{tile_id}' - - -def _stage_tile( - *, - tile_id: str, - work_dir: str, - gcp_project: str, - force: bool, - mirror_dataset: str, - gcs_bucket: str, - gcs_staging_prefix: str, - asset_root: str, - resolve_tile_url: Callable[[str], str], -) -> tuple[str, str, str, str]: - """Phase A worker: download via mirror + upload to GCS. - - Returns ``(tile_id, gcs_uri, local_path, blob_name)``. ``force=True`` - additionally pre-deletes the per-tile asset so the subsequent ingest - starts from a clean slate (paired with ``allow_overwrite=True`` on - submission for resilience against eventual-consistency reads). - """ - local_path = os.path.join(work_dir, f'{tile_id}.tif') - - # Mirror is keyed on tile_id (stable), so rotated source URLs never - # invalidate a cached tile. The closure defers URL resolution to - # mirror-miss. - def _resolve(tile: str = tile_id) -> str: - return resolve_tile_url(tile) - - fetch_with_mirror( - local_path, - dataset=mirror_dataset, - filename=f'{tile_id}.tif', - gcp_project=gcp_project, - source=_resolve, - ) - - blob_name = f'{gcs_staging_prefix}/{tile_id}.tif' - if force: - delete_asset_if_present(_tile_image_asset_id(asset_root, tile_id)) - gcs_uri = upload_to_gcs(gcp_project, gcs_bucket, blob_name, local_path) - return tile_id, gcs_uri, local_path, blob_name - - -def _delete_collection_if_present( - *, - dataset_label: str, - asset_root: str, - default_tile_ids: tuple[str, ...] | list[str], -) -> None: - """Delete the ImageCollection and all member images. - - Collections must be empty before deletion, so member images are - dropped first. Iterates ``default_tile_ids`` (the canonical CONUS - tile list) — additional tiles outside this list would survive, but - a future re-extract under ``force=True`` would re-ingest cleanly - anyway. - """ - if not asset_exists(asset_root): - return - logger.info(f'{dataset_label}: force=True, clearing {asset_root}') - for tile_id in default_tile_ids: - delete_asset_if_present(_tile_image_asset_id(asset_root, tile_id)) - delete_asset_if_present(asset_root) - - -def run_tile_collection_extract( - *, - dataset_label: str, - mirror_dataset: str, - gcs_bucket: str, - gcs_staging_prefix: str, - asset_root: str, - band_name: str, - gcp_project: str, - force: bool, - default_tile_ids: tuple[str, ...] | list[str], - tile_ids: list[str] | tuple[str, ...] | None, - resolve_tile_url: Callable[[str], str], - staging_max_workers: int = 16, -) -> str: - """Run the canonical Phase A/B/C/D tile-collection extract. - - Per-tile ingestion produces an Image asset under - ``{asset_root}/tile_{tile_id}``; the collection itself is created - via ``create_image_collection_if_absent`` if absent. The transform - consumer lazy-mosaics the collection on read. - - Args: - dataset_label: Human-readable log prefix (e.g. ``'Harris AGB'``, - ``'GFW Peatlands'``). - mirror_dataset: Mirror-bucket key passed to - ``fetch_with_mirror`` (e.g. ``'harris_agb'``). - gcs_bucket: GCS bucket for tile staging. - gcs_staging_prefix: GCS path prefix (e.g. - ``'luc_high_res/staging/harris_agb'``). - asset_root: GEE ImageCollection asset path. - band_name: Band name to assign on ingestion. - gcp_project: GCP project for GCS + EE. - force: If True, re-ingest existing tiles. - default_tile_ids: Canonical CONUS tile list, used when - ``tile_ids`` is None (and to enumerate the collection for - the full-rebuild force path). - tile_ids: Override tile list (tests / partial rebuilds). - resolve_tile_url: Callable producing the per-tile download URL. - staging_max_workers: Phase A pool size (default 16). - - Returns: - ``asset_root``. Returned even on partial failure — callers - should re-run with ``force=True`` to retry. - """ - tiles: tuple[str, ...] | list[str] = ( - tile_ids if tile_ids is not None else default_tile_ids - ) - run_id = uuid.uuid4().hex[:8] - full_rebuild = tile_ids is None - - if force and full_rebuild: - # Default-tile-list rebuild: drop the whole collection so the - # run starts from a clean slate (also handles the case where - # the collection got into a weird partial state from prior - # crashes). - _delete_collection_if_present( - dataset_label=dataset_label, - asset_root=asset_root, - default_tile_ids=default_tile_ids, - ) - elif force: - # Subset rebuild: only delete the tiles the caller actually - # asked to rebuild. Nuking the parent collection (or sibling - # tiles) would silently destroy unrelated data — historically - # caused an integration test to wipe 18 of 19 production tiles. - for tid in tiles: - delete_asset_if_present(_tile_image_asset_id(asset_root, tid)) - create_image_collection_if_absent(asset_root) - - tiles_to_extract = [ - tid - for tid in tiles - if force or not asset_exists(_tile_image_asset_id(asset_root, tid)) - ] - if not tiles_to_extract: - logger.info(f'{dataset_label}: all {len(tiles)} tile(s) cached, nothing to do') - return asset_root - logger.info( - f'{dataset_label}: {len(tiles_to_extract)}/{len(tiles)} tile(s) need ' - f'extraction; run_id={run_id}' - ) - - work_dir = tempfile.mkdtemp(prefix=f'{mirror_dataset}_{run_id}_') - staged: list[tuple[str, str, str, str]] = [] - task_to_asset: dict[str, str] = {} - task_to_blob: dict[str, str] = {} - # Blobs that made it to GCS but never got a successful ingest task - # ID (Phase B exception), so Phase D still cleans them up. - orphan_blobs: list[str] = [] - - try: - # Phase A: concurrent staging. - with ThreadPoolExecutor(max_workers=staging_max_workers) as ex: - futures = { - ex.submit( - _stage_tile, - tile_id=tid, - work_dir=work_dir, - gcp_project=gcp_project, - force=force, - mirror_dataset=mirror_dataset, - gcs_bucket=gcs_bucket, - gcs_staging_prefix=gcs_staging_prefix, - asset_root=asset_root, - resolve_tile_url=resolve_tile_url, - ): tid - for tid in tiles_to_extract - } - for fut in as_completed(futures): - tid = futures[fut] - try: - staged.append(fut.result()) - except Exception as exc: - logger.error(f'{dataset_label}: staging failed for {tid}: {exc}') - - # Phase B: synchronous batch submission. Each staged tile - # becomes one ingestion task; failures are isolated — siblings - # still go. - for tile_id, gcs_uri, _local, blob_name in staged: - asset_id = _tile_image_asset_id(asset_root, tile_id) - try: - task_id = start_ingestion_no_wait( - gcs_uri, - asset_id, - band_name=band_name, - allow_overwrite=force, - ) - except Exception as exc: - logger.error(f'{dataset_label}: submit failed for {tile_id}: {exc}') - orphan_blobs.append(blob_name) - continue - task_to_asset[task_id] = asset_id - task_to_blob[task_id] = blob_name - - # Phase C: bulk poll. - if task_to_asset: - failed = wait_for_tasks( - list(task_to_asset), - asset_ids_for_logging=task_to_asset, - ) - if failed: - logger.warning( - f'{dataset_label}: {len(failed)}/{len(task_to_asset)} ' - f'tile(s) failed to ingest; collection at {asset_root} ' - f'is partial — re-run with force=True to retry' - ) - finally: - # Phase D: cleanup — GCS blobs first, then local files, then - # work dir. - for blob_name in list(task_to_blob.values()) + orphan_blobs: - try: - delete_gcs_blob(gcp_project, gcs_bucket, blob_name) - except Exception as exc: - logger.warning( - f'{dataset_label}: blob cleanup failed for {blob_name}: ' f'{exc}' - ) - shutil.rmtree(work_dir, ignore_errors=True) - - logger.info(f'{dataset_label}: extract complete → {asset_root}') - return asset_root diff --git a/jdluc/extract/county_fips.py b/jdluc/extract/county_fips.py deleted file mode 100644 index 8670f1f..0000000 --- a/jdluc/extract/county_fips.py +++ /dev/null @@ -1,172 +0,0 @@ -"""CONUS county-FIPS label raster extract. - -Builds a single-band int32 image at the GLAD 0.00025° grid by painting -the GEOID property of every CONUS feature in ``TIGER/2018/Counties`` -into ``ee.Image.constant(0).int()`` and ``selfMask()``-ing the -unpainted background. Each pixel inside a CONUS county carries the -county's 5-digit FIPS as an integer (e.g. Polk County, IA → 19153); -pixels outside CONUS or in AK / HI / territories are masked. - -``paint(FC, value)`` implements the pixel-center rasterization rule at -GLAD 30m, near-identical to a majority-area rule (XOR < 0.003%). - -The extract is server-side only (no HTTP, no GCS staging): TIGER -counties are a GEE-native FeatureCollection and the export is a -single ``Export.image.toAsset`` task pinned to GLAD CRS / CRS_TRANSFORM -over the bounding rectangle of the CONUS county union. - -The asset is the single source of geographic truth for the transform -pipeline: polygon mask via ``fips.mask()``, per-county grouping band -via ``fips`` itself. -""" - -import logging - -import ee - -from jdluc.utils._ee_types import EEFeature, EEFeatureCollection, EEGeometry, EEImage -from jdluc.utils.constants import ( - CONUS_STATE_FIPS, - GEE_COUNTY_FIPS_LABEL, - GEE_TIGER_COUNTIES, - GLAD_CRS, - GLAD_CRS_TRANSFORM, -) -from jdluc.utils.gee import ( - asset_exists, - delete_asset_if_present, - wait_for_export_task, -) - -logger = logging.getLogger(__name__) - -# Output band name; the transform stage selects this when consuming the -# asset. -BAND_NAME: str = 'county_fips' - -# Property attached server-side: integer parse of TIGER's GEOID string. -# GEOID is STATEFP+COUNTYFP zero-padded (e.g. "19153"); parsing as int -# yields the methodology-standard FIPS code (1001..56045 across CONUS). -_FIPS_PROPERTY: str = 'fips_int' - - -def _tag_with_fips_int(feature: EEFeature) -> EEFeature: - """Attach an int-typed `fips_int` property derived from GEOID. - - GEOID is TIGER's zero-padded 5-digit STATEFP+COUNTYFP string; - parsing as int yields the methodology FIPS code (1001..56045). - Pulled out as a named helper rather than an inline lambda so the - server-side `ee.Number.parse(ee.String(...)).toInt()` chain is - unit-testable in isolation. - """ - return feature.set( - _FIPS_PROPERTY, - ee.Number.parse(ee.String(feature.get('GEOID'))).toInt(), - ) - - -def _build_conus_counties_fc() -> EEFeatureCollection: - """Filter TIGER counties to CONUS and tag each feature with `fips_int`. - - Filtering and tagging stay server-side; the python list of state - FIPS codes is just used to build the inList filter. Tagging adds - the int-typed FIPS property `paint` reads in `_build_fips_image`. - """ - return ( - ee.FeatureCollection(GEE_TIGER_COUNTIES) - .filter(ee.Filter.inList('STATEFP', CONUS_STATE_FIPS)) - .map(_tag_with_fips_int) - ) - - -def _build_fips_image(counties_fc: EEFeatureCollection) -> EEImage: - """Paint the FIPS int property into a self-masked int32 image. - - The painted-and-masked image is the asset's content. ``selfMask()`` - drops the unpainted background (value=0); since the smallest CONUS - FIPS is 1001 (Autauga, AL), no real county pixel is ever 0, so the - mask is unambiguous. - - No inline ``.reproject()``: per the investigation doc § "Bisect - probe: inline paint doesn't amortize, asset-loaded mask does", - ``.reproject()`` re-evaluates the polygon paint per tile rather - than materializing once, so it adds cost to every consumer call - against an un-materialized graph. The grid is instead pinned at - the export sink via the ``crs`` / ``crsTransform`` arguments, so - the on-disk asset is GLAD-aligned and downstream transform reads - come from the materialized asset (no per-tile paint re-evaluation). - """ - return ( - ee.Image.constant(0) - .int() - .paint(counties_fc, _FIPS_PROPERTY) - .selfMask() - .rename(BAND_NAME) - ) - - -def _start_fips_export( - image: EEImage, asset_id: str, region: EEGeometry, description: str -) -> ee.batch.Task: - """Submit the GLAD-grid asset export and return the task handle. - - Pins ``crs`` and ``crsTransform`` to GLAD's so the produced asset - is byte-aligned with every other transform-stage output and no - consumer ever pays a per-tile reprojection at read time. - """ - task = ee.batch.Export.image.toAsset( - image=image, - description=description, - assetId=asset_id, - region=region, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e13), - ) - task.start() - logger.info( - f'county_fips: export task started (asset={asset_id}, ' - f'description={description})' - ) - return task - - -def extract_county_fips(gcp_project: str, force: bool = False) -> str: - """Materialize the CONUS county-FIPS label asset and return its ID. - - Args: - gcp_project: GCP project. Accepted for orchestrator-API parity - with other extractors; this extract is server-side only and - does not touch GCS or HTTP, so the project is unused inside - this function (auth is the caller's responsibility, mirroring - the other extractors). - force: If True, delete any existing target asset before re-export. - - Returns: - The materialized GEE asset ID (= ``GEE_COUNTY_FIPS_LABEL``). - """ - del gcp_project # API parity; see docstring. - - if force: - delete_asset_if_present(GEE_COUNTY_FIPS_LABEL) - - counties_fc = _build_conus_counties_fc() - image = _build_fips_image(counties_fc) - region = counties_fc.geometry().bounds() - - task = _start_fips_export( - image=image, - asset_id=GEE_COUNTY_FIPS_LABEL, - region=region, - description='extract_county_fips_conus', - ) - wait_for_export_task(task, GEE_COUNTY_FIPS_LABEL) - - if not asset_exists(GEE_COUNTY_FIPS_LABEL): - raise RuntimeError( - f'county_fips: export task completed but asset is not present: ' - f'{GEE_COUNTY_FIPS_LABEL}' - ) - - logger.info(f'county_fips: extract complete → {GEE_COUNTY_FIPS_LABEL}') - return GEE_COUNTY_FIPS_LABEL diff --git a/jdluc/extract/extract.py b/jdluc/extract/extract.py deleted file mode 100644 index c67aed7..0000000 --- a/jdluc/extract/extract.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Extract stage entry point. - -Orchestrates ingestion of all external non-GEE-native datasets into Google -Earth Engine. Iterates through ``NON_NATIVE_DATASETS``, compares each -dataset's ``expected_version`` against what's already in GEE, and calls the -per-dataset extractor on a miss. - -GEE-native datasets (GLAD GLC, CDL, SoilGrids, TIGER states, TIGER -counties) are used directly and require no ingestion, so they do not -appear in ``NON_NATIVE_DATASETS`` and are never touched by this module. - -Extracted assets are tagged with an upstream-dataset version suffix -(e.g. ``harris_agb_conus_v2021``) — no code-SHA suffix, since extract -logic is stable and re-running at the same upstream version produces -byte-identical assets. - -See specs/pipeline_tech_design.md § Extract for details. -""" - -import dataclasses -import logging -from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor, as_completed -from typing import Protocol - -from jdluc.extract import ( - county_fips, - gfw_peatlands, - harris_agb, - huang_bgb, - ipcc_climate_zones, - nass_yields, -) -from jdluc.utils.constants import DATASET_INVENTORY, NON_NATIVE_DATASETS -from jdluc.utils.gee import asset_is_populated - -logger = logging.getLogger(__name__) - - -class ExtractorCallable(Protocol): - def __call__(self, gcp_project: str, force: bool = False) -> str: ... - - -@dataclasses.dataclass -class ExtractResult: - cached: list[str] = dataclasses.field(default_factory=list) - extracted: list[str] = dataclasses.field(default_factory=list) - failed: dict[str, str] = dataclasses.field(default_factory=dict) - - -class ExtractError(Exception): - def __init__(self, result: ExtractResult) -> None: - self.result = result - failed_summary = ', '.join(f'{k}({v})' for k, v in result.failed.items()) - super().__init__( - f'Extract failed for {len(result.failed)} dataset(s): {failed_summary}' - ) - - -DATASET_FAMILY_TO_EXTRACTOR = { - 'harris_agb': harris_agb.extract_harris_agb, - 'huang_bgb': huang_bgb.extract_huang_bgb, - 'nass_yields': nass_yields.extract_nass_yields, - 'ipcc_climate_zones': ipcc_climate_zones.extract_ipcc_climate_zones, - 'gfw_peatlands': gfw_peatlands.extract_gfw_peatlands, - 'county_fips': county_fips.extract_county_fips, -} - - -def extract_all( - gcp_project: str, - force: bool = False, - *, - extractor_factory: Callable[ - [str], ExtractorCallable - ] = DATASET_FAMILY_TO_EXTRACTOR.__getitem__, - asset_exists: Callable[[str], bool] = asset_is_populated, -) -> ExtractResult: - """Run every non-native dataset extractor, skipping cache hits. - - Args: - gcp_project: GCP project ID for Earth Engine client initialization - inside per-dataset extractors. Caller is responsible for having - initialized ``ee`` already (see ``run_pipeline``). - force: If True, ignore existing assets and re-ingest unconditionally. - extractor_factory: Dispatch from dataset family to its extractor function. - asset_exists: Override for the cache-hit probe. Injectable for unit - testing; defaults to ``utils.gee.asset_is_populated`` so that an - empty ImageCollection (e.g. left behind by a partial tile - ingestion) is treated as a miss, not a hit. - - Returns: - ExtractResult with ``cached`` / ``extracted`` / ``failed`` populated. - Failures do NOT abort the orchestrator — other datasets still run, - so a single broken source doesn't block unrelated work. - """ - result = ExtractResult() - - # Step 1: probe cache and split into cached / work queues. The probes - # are cheap getAsset calls and run sequentially — keeps the log order - # for cache-hit reporting deterministic. - work: list[str] = [] - for dataset_family in NON_NATIVE_DATASETS: - entry = DATASET_INVENTORY[dataset_family] - asset_id = entry['gee_asset_id'] - expected_version = entry.get('expected_version', '') - - if not force and asset_exists(asset_id): - logger.info( - f'extract[{dataset_family}]: cache hit at ' - f'{asset_id} (version={expected_version})' - ) - result.cached.append(dataset_family) - continue - - logger.info( - f'extract[{dataset_family}]: cache miss, ' - f'running extractor (version={expected_version}, force={force})' - ) - work.append(dataset_family) - - # Step 2: fan out the cache-misses in parallel. The 6 extractors have - # no inter-dependencies; each writes a distinct GEE asset and handles - # its own internal threading. The GEE per-project ingest task cap (~20) - # is the binding upstream constraint, well above 6 outer workers. - outcomes: dict[str, str | Exception] = {} - if work: - futures: dict[Future[str], str] = {} - with ThreadPoolExecutor(max_workers=len(work)) as executor: - for dataset_family in work: - extractor = extractor_factory(dataset_family) - futures[executor.submit(extractor, gcp_project, force)] = dataset_family - for future in as_completed(futures): - family = futures[future] - try: - outcomes[family] = future.result() - except Exception as exc: - outcomes[family] = exc - - # Step 3: collate in NON_NATIVE_DATASETS order so result.extracted / - # result.failed are deterministic regardless of completion order. - for dataset_family in NON_NATIVE_DATASETS: - if dataset_family not in outcomes: - continue - outcome = outcomes[dataset_family] - asset_id = DATASET_INVENTORY[dataset_family]['gee_asset_id'] - if isinstance(outcome, Exception): - logger.error(f'extract[{dataset_family}]: failed — {outcome}') - result.failed[dataset_family] = str(outcome) - continue - if outcome != asset_id: - logger.warning( - f'extract[{dataset_family}]: extractor returned ' - f'{outcome!r} but inventory expects {asset_id!r}' - ) - logger.info(f'extract[{dataset_family}]: extracted → {outcome}') - result.extracted.append(dataset_family) - - logger.info( - f'extract_all: cached={result.cached}, ' - f'extracted={result.extracted}, failed={list(result.failed)}' - ) - return result diff --git a/jdluc/extract/gfw_peatlands.py b/jdluc/extract/gfw_peatlands.py deleted file mode 100644 index df7a0eb..0000000 --- a/jdluc/extract/gfw_peatlands.py +++ /dev/null @@ -1,138 +0,0 @@ -"""GFW Global Peatlands raster extract (CC BY 4.0 composite). - -Downloads 19 CONUS GeoTIFF tiles from Global Forest Watch's Global -Peatlands dataset, stages each to GCS, and ingests them as Image -members of an ``ImageCollection`` asset. The transform consumer -(``transform/land_use.py::_load_is_peatland``) lazy-mosaics the -collection on read. - -The orchestrator (Phases A/B/C/D) lives in -``extract/_tile_pipeline.py``; this module supplies the GFW-specific -URL template + dataset constants. - -GFW's product is a composite raster: - -- Above 40°N: Xu et al. (2018) PEATMAP -- Below 40°N (default): Gumbricht et al. (2017) tropical wetlands/peatlands -- Indonesia/Malaysia override: Miettinen et al. (2016) -- Lowland Peruvian Amazon override: Hastie et al. (2022) -- Congo basin override: Crezee et al. (2022) - -All layers are rasterized to the 30 m Hansen Global Forest Change -grid and merged into a single raster, distributed as 10°×10° tiles -on the same grid identifiers as Harris AGB. CONUS coverage is split -across the 40°N cutoff: tiles fully above 40°N (the northern tier -through the Corn Belt and Great Plains) read from Xu PEATMAP; -tiles below 40°N (the mid-Atlantic, Southeast, southern California, -Arizona, New Mexico, and most of Texas) fall through to Gumbricht. -This is a known limitation — Gumbricht is tropical-tuned and a poor -fit for temperate-US peatlands; see `specs/methodology.md` Appendix 2 -(Peatland dataset choice) and `specs/analyses/orbae_de_corn_comparison.ipynb` -for the empirical impact. - -The tile-list (``CONUS_TILE_IDS``) is GFW-Peatlands-specific: GFW omits -tiles with no peatland pixels, so two of Harris AGB's 21 CONUS -tiles (30N_070W in the Caribbean, 30N_130W in the Pacific) don't -exist in the manifest. The 19-tile constant below is hard-coded -against the v20230315 manifest the user downloaded; if GFW ever -republishes with denser coverage, regenerate from the new CSV. - -Source: https://data.globalforestwatch.org/datasets/gfw::global-peatlands -License: CC BY 4.0. - -API key (``GFW_DATA_API_KEY`` in ``utils/constants.py``): the key -ships embedded in any user's CSV download from the GFW Open Data -Portal — it's a public rate-limited token, not a secret. If the key -401s in the future, re-download the CSV from the portal and bump -the constant in ``utils/constants.py``. -""" - -from jdluc.extract._tile_pipeline import run_tile_collection_extract - -# GCS staging. Tiles live at ``{GCS_PATH}/{tile_id}.tif``. -from jdluc.utils.constants import ( - GCS_BUCKET_NAME, - GEE_GFW_PEATLANDS, - GFW_DATA_API_KEY, - GFW_PEATLANDS_URL_TEMPLATE, -) - -GCS_STAGING_PREFIX: str = 'luc_high_res/staging/gfw_peatlands' - -BAND_NAME: str = 'is_peatland' - -# CONUS tiles GFW Global Peatlands actually publishes. The full Harris -# AGB grid has 21 CONUS tiles, but GFW omits tiles with no peatland -# pixels — 30N_070W (Caribbean, no land) and 30N_130W (Pacific, off -# coast) are absent from the v20230315 manifest. Hard-coded against -# the user's downloaded CSV manifest rather than re-derived from -# Harris AGB's grid; we are version-pinned to v20230315 anyway. If -# GFW ever republishes with denser coverage, regenerate from the new -# CSV manifest. -CONUS_TILE_IDS: tuple[str, ...] = ( - '30N_080W', - '30N_090W', - '30N_100W', - '30N_110W', - '30N_120W', - '40N_070W', - '40N_080W', - '40N_090W', - '40N_100W', - '40N_110W', - '40N_120W', - '40N_130W', - '50N_070W', - '50N_080W', - '50N_090W', - '50N_100W', - '50N_110W', - '50N_120W', - '50N_130W', -) - -__all__ = ('CONUS_TILE_IDS', 'extract_gfw_peatlands') - - -def _resolve_tile_url(tile_id: str) -> str: - """Format the GFW per-tile GeoTIFF download URL. - - Unlike Harris AGB (which discovers per-tile signed URLs via an - ArcGIS FeatureServer query), GFW exposes a stable URL template - keyed on ``tile_id`` and a public API key, so the URL is - template-constructed without any HTTP probe. - """ - return GFW_PEATLANDS_URL_TEMPLATE.format(tile_id=tile_id, api_key=GFW_DATA_API_KEY) - - -def extract_gfw_peatlands( - gcp_project: str, - force: bool = False, - *, - tile_ids: list[str] | tuple[str, ...] | None = None, -) -> str: - """Download GFW Global Peatlands tiles, stage to GCS, ingest into GEE. - - Args: - gcp_project: GCP project for GCS + EE. - force: If True, delete and re-ingest any existing tiles. - tile_ids: Override tile list (tests). Defaults to CONUS_TILE_IDS. - - Returns: - GEE ImageCollection asset ID. Returned even when individual tiles - failed to ingest — callers must inspect the collection and re-run - with ``force=True`` to retry. - """ - return run_tile_collection_extract( - dataset_label='GFW Peatlands', - mirror_dataset='gfw_peatlands', - gcs_bucket=GCS_BUCKET_NAME, - gcs_staging_prefix=GCS_STAGING_PREFIX, - asset_root=GEE_GFW_PEATLANDS, - band_name=BAND_NAME, - gcp_project=gcp_project, - force=force, - default_tile_ids=CONUS_TILE_IDS, - tile_ids=tile_ids, - resolve_tile_url=_resolve_tile_url, - ) diff --git a/jdluc/extract/harris_agb.py b/jdluc/extract/harris_agb.py deleted file mode 100644 index 005b5f8..0000000 --- a/jdluc/extract/harris_agb.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Harris et al. (2021) above-ground live woody biomass extract. - -Discovers 19 CONUS tiles via Global Forest Watch's ArcGIS FeatureServer, -downloads each GeoTIFF, stages it to GCS, and ingests each as an Image -member of an ``ImageCollection`` asset. The orchestrator (Phases A/B/C/D) -lives in ``extract/_tile_pipeline.py``; this module supplies the -Harris-specific URL discovery + dataset constants. - -Source: Harris et al. (2021) *Global maps of twenty-first century forest -carbon fluxes*. Nature Climate Change, 11, 234–240. -DOI 10.1038/s41558-020-00976-6. -Portal: -https://data.globalforestwatch.org/datasets/gfw::aboveground-live-woody-biomass-density -""" - -from typing import Any - -import requests - -from jdluc.extract._tile_pipeline import run_tile_collection_extract -from jdluc.utils.constants import ( - GCS_BUCKET_NAME, - GEE_HARRIS_AGB, - HARRIS_AGB_ARCGIS_FEATURESERVER, -) - -# 10°×10° tiles covering CONUS landmass. Tile IDs are the upper-left -# corner; e.g. 40N_080W covers 30°N–40°N, 80°W–70°W (includes Delaware). -# The full 3×7 grid would be 21 tiles, but Harris AGB's FeatureServer -# omits 30N_070W (Caribbean) and 30N_130W (Pacific) — both ocean with -# no biomass — so we list only the 19 tiles the FeatureServer actually -# publishes. (GFW's peatlands manifest has the same gap; see -# extract/gfw_peatlands.py.) -CONUS_TILE_IDS: list[str] = [ - f'{row}_{col}' - for row in ('30N', '40N', '50N') - for col in ('070W', '080W', '090W', '100W', '110W', '120W', '130W') - if (row, col) not in {('30N', '070W'), ('30N', '130W')} -] - -# GCS staging. Tiles live at ``{GCS_PATH}/{tile_id}.tif``. -GCS_STAGING_PREFIX: str = 'luc_high_res/staging/harris_agb' - -BAND_NAME: str = 'agb_mg_ha' - - -def _fetch_tile_download_url(tile_id: str, *, timeout_s: float = 30.0) -> str: - """Resolve ``tile_id`` to its ArcGIS-signed download URL.""" - params = { - 'where': f"tile_id='{tile_id}'", - 'outFields': 'tile_id,Mg_ha_1_download', - 'f': 'json', - } - response = requests.get( - HARRIS_AGB_ARCGIS_FEATURESERVER, params=params, timeout=timeout_s - ) - response.raise_for_status() - features = response.json().get('features', []) - if not features: - raise RuntimeError(f'Harris AGB: tile {tile_id!r} not found on FeatureServer') - url = features[0]['attributes'].get('Mg_ha_1_download') - if not url: - raise RuntimeError( - f'Harris AGB: FeatureServer has no Mg_ha_1_download for {tile_id!r}' - ) - return str(url) - - -def extract_harris_agb( - gcp_project: str, - force: bool = False, - *, - tile_ids: list[str] | None = None, - feature_server_params: dict[str, Any] | None = None, -) -> str: - """Discover, download, stage, and ingest Harris AGB tiles into GEE. - - Args: - gcp_project: GCP project for GCS + EE. - force: If True, delete and re-ingest any existing tiles. - tile_ids: Override tile list (tests). Defaults to CONUS_TILE_IDS. - - Returns: - GEE ImageCollection asset ID. Returned even when individual tiles - failed to ingest — callers must inspect the collection and re-run - with ``force=True`` to retry. (The orchestrator's - ``asset_is_populated`` cache check sees a partial collection as - populated, matching the behavior for any non-empty IC.) - """ - return run_tile_collection_extract( - dataset_label='Harris AGB', - mirror_dataset='harris_agb', - gcs_bucket=GCS_BUCKET_NAME, - gcs_staging_prefix=GCS_STAGING_PREFIX, - asset_root=GEE_HARRIS_AGB, - band_name=BAND_NAME, - gcp_project=gcp_project, - force=force, - default_tile_ids=CONUS_TILE_IDS, - tile_ids=tile_ids, - resolve_tile_url=_fetch_tile_download_url, - ) diff --git a/jdluc/extract/huang_bgb.py b/jdluc/extract/huang_bgb.py deleted file mode 100644 index 3fe961d..0000000 --- a/jdluc/extract/huang_bgb.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Huang et al. (2021) below-ground biomass extract. - -Fetches ``data_code_to_submit.zip`` from Figshare (DOI -10.6084/m9.figshare.12199637.v1), extracts the ``pergridarea_bgb.nc`` -NetCDF, writes a CONUS-clipped GeoTIFF, stages it to GCS, and ingests it -as a single ``Image`` asset. CRS reprojection and NoData gap-fill are -NOT done here — ``transform/emissions.py::_load_bgb`` owns that grid -work so the extract output stays as close to the source as possible. -""" - -import logging -import os -import shutil -import tempfile -import uuid -import zipfile - -import rioxarray # noqa: F401 — registers the .rio accessor on DataArray. -import xarray as xr - -from jdluc.extract.mirror import fetch_with_mirror -from jdluc.utils.constants import ( - GCS_BUCKET_NAME, - GEE_HUANG_BGB, - HUANG_BGB_FIGSHARE_URL, -) -from jdluc.utils.gee import ( - delete_asset_if_present, - delete_gcs_blob, - start_ingestion_and_wait, - upload_to_gcs, -) - -logger = logging.getLogger(__name__) - -# CONUS bounding box (with small buffer). -CONUS_WEST: float = -130.0 -CONUS_EAST: float = -65.0 -CONUS_SOUTH: float = 24.0 -CONUS_NORTH: float = 50.0 - - -GCS_STAGING_PREFIX: str = 'luc_high_res/staging/huang_bgb' - -NETCDF_MEMBER_NAME: str = 'pergridarea_bgb.nc' -# AROOT is Huang's below-ground biomass density variable (Mg/ha). -NETCDF_VARIABLE: str = 'AROOT' - -BAND_NAME: str = 'bgb_mg_ha' - - -def _write_conus_geotiff(netcdf_path: str, geotiff_path: str) -> None: - """Read NetCDF, select AROOT, clip to CONUS bounds, write GeoTIFF. - - No reproject, no NoData fill — downstream transform handles both. - """ - - logger.info(f'Huang BGB: loading NetCDF {netcdf_path}') - with xr.open_dataset(netcdf_path) as ds: - da = ds[NETCDF_VARIABLE].sel( - LAT=slice(CONUS_SOUTH, CONUS_NORTH), - LON=slice(CONUS_WEST, CONUS_EAST), - ) - da = da.rio.set_spatial_dims(x_dim='LON', y_dim='LAT') - da = da.rio.write_crs('EPSG:4326') - logger.info( - f'Huang BGB: clipped shape={tuple(da.shape)}, ' - f'lat=[{float(da.LAT.min()):.4f}, {float(da.LAT.max()):.4f}], ' - f'lon=[{float(da.LON.min()):.4f}, {float(da.LON.max()):.4f}]' - ) - da.rio.to_raster(geotiff_path, compress='LZW', dtype='float32') - - -def _extract_netcdf(zip_path: str, work_dir: str) -> str: - """Extract ``NETCDF_MEMBER_NAME`` from ``zip_path`` into ``work_dir``.""" - logger.info(f'Huang BGB: extracting {NETCDF_MEMBER_NAME} from {zip_path}') - with zipfile.ZipFile(zip_path) as zf: - candidates = [n for n in zf.namelist() if n.endswith(NETCDF_MEMBER_NAME)] - if not candidates: - raise RuntimeError( - f'Huang BGB: {NETCDF_MEMBER_NAME!r} not found in archive at ' - f'{zip_path}; archive members: {zf.namelist()[:10]}...' - ) - zf.extract(candidates[0], work_dir) - return os.path.join(work_dir, candidates[0]) - - -def extract_huang_bgb(gcp_project: str, force: bool = False) -> str: - """Fetch Huang BGB, clip to CONUS, stage to GCS, ingest into GEE. - - Args: - gcp_project: GCP project for GCS + EE. - force: If True, delete + re-ingest the existing asset. - - Returns: - GEE Image asset ID. - """ - run_id = uuid.uuid4().hex[:8] - - if force: - delete_asset_if_present(GEE_HUANG_BGB) - - work_dir = tempfile.mkdtemp(prefix=f'huang_bgb_{run_id}_') - logger.info(f'Huang BGB: work_dir={work_dir}') - - zip_path = os.path.join(work_dir, 'data_code_to_submit.zip') - geotiff_path = os.path.join(work_dir, 'huang_bgb_conus.tif') - blob_name = f'{GCS_STAGING_PREFIX}/{run_id}/huang_bgb_conus.tif' - - try: - fetch_with_mirror( - zip_path, - dataset='huang_bgb', - filename='data_code_to_submit.zip', - gcp_project=gcp_project, - source=HUANG_BGB_FIGSHARE_URL, - timeout_s=900.0, - ) - netcdf_path = _extract_netcdf(zip_path, work_dir) - _write_conus_geotiff(netcdf_path, geotiff_path) - - gcs_uri = upload_to_gcs(gcp_project, GCS_BUCKET_NAME, blob_name, geotiff_path) - try: - start_ingestion_and_wait( - gcs_uri, - GEE_HUANG_BGB, - band_name=BAND_NAME, - allow_overwrite=force, - ) - finally: - delete_gcs_blob(gcp_project, GCS_BUCKET_NAME, blob_name) - finally: - shutil.rmtree(work_dir, ignore_errors=True) - - logger.info(f'Huang BGB: extract complete → {GEE_HUANG_BGB}') - return GEE_HUANG_BGB diff --git a/jdluc/extract/ipcc_climate_zones.py b/jdluc/extract/ipcc_climate_zones.py deleted file mode 100644 index cd367e3..0000000 --- a/jdluc/extract/ipcc_climate_zones.py +++ /dev/null @@ -1,169 +0,0 @@ -"""IPCC 2006 climate zone raster extract. - -Fetches the single 0.5° GeoTIFF from Zenodo record 7303808, stages it -to GCS, and ingests it at a scratch native-projection asset. A server- -side ``Export.image.toAsset`` then reprojects onto the GLAD 0.00025° -grid via nearest-neighbor (the source is categorical zone codes) and -writes the final versioned asset. The scratch asset and GCS staging -object are removed on success. - -Precomputing the reprojection here (rather than at read time in -``transform/emissions.py::_load_ipcc_climate``) keeps the GLAD-grid -transform out of every pipeline run — the 0.5° → 0.00025° lift is -non-trivial and the output is static. -""" - -import logging -import os -import shutil -import tempfile -import uuid - -import ee -import requests - -from jdluc.extract.mirror import fetch_with_mirror -from jdluc.utils.constants import ( - GCS_BUCKET_NAME, - GEE_ASSET_ROOT, - GEE_IPCC_CLIMATE_ZONES, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - IPCC_CLIMATE_ZONES_ZENODO_URL, -) -from jdluc.utils.gee import ( - asset_exists, - delete_asset_if_present, - delete_gcs_blob, - start_ingestion_and_wait, - upload_to_gcs, - wait_for_export_task, -) - -logger = logging.getLogger(__name__) - - -GCS_STAGING_BLOB: str = 'luc_high_res/staging/ipcc_climate_zones_v2006.tif' - -# Stable local + mirror filename. The upstream Zenodo filename could -# drift across record revisions; we use our own name so the mirror key -# is stable and the Zenodo probe is skipped on cache hit. -SOURCE_FILENAME: str = 'ipcc_climate_zones_v2006.tif' - -# Scratch asset holds the ingested native-resolution TIF; the final -# asset is produced by a server-side GEE export that reprojects onto -# the GLAD grid. -SCRATCH_ASSET_ID: str = f'{GEE_ASSET_ROOT}/ipcc_climate_zones_v2006_native' - -BAND_NAME: str = 'ipcc_climate_zone' - - -def _discover_zenodo_tif_url() -> tuple[str, str]: - """Resolve the IPCC climate-zone file URL on the Zenodo record. - - Returns ``(filename, download_url)``. Uses the record-level API - endpoint stored in ``IPCC_CLIMATE_ZONES_ZENODO_URL`` — the record - contains exactly one .tif file; fail fast if that changes. - """ - logger.info(f'Fetching Zenodo record: {IPCC_CLIMATE_ZONES_ZENODO_URL}') - response = requests.get(IPCC_CLIMATE_ZONES_ZENODO_URL, timeout=60) - response.raise_for_status() - files = response.json().get('files', []) - tifs = [f for f in files if str(f.get('key', '')).lower().endswith('.tif')] - if len(tifs) != 1: - keys = [f.get('key') for f in files] - raise RuntimeError( - f'Zenodo record expected to contain exactly 1 .tif; found {keys}' - ) - entry = tifs[0] - return entry['key'], entry['links']['self'] - - -def _start_reproject_export(src_asset_id: str, dst_asset_id: str) -> ee.batch.Task: - """Reproject src onto the GLAD grid and export to dst (server-side). - - Default GEE resampling for integer-typed rasters is nearest-neighbor, - which is what this categorical raster needs. We still call - ``.reproject(...)`` explicitly so the intent is visible at the call - site. - """ - src = ee.Image(src_asset_id).reproject( - crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM - ) - global_region = ee.Geometry.Rectangle( - coords=[-180, -90, 180, 90], proj=GLAD_CRS, geodesic=False - ) - task = ee.batch.Export.image.toAsset( - image=src, - description='ipcc_climate_zones_v2006_reproject', - assetId=dst_asset_id, - region=global_region, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e13), - ) - task.start() - logger.info(f'Reproject export task started: {dst_asset_id}') - return task - - -def extract_ipcc_climate_zones(gcp_project: str, force: bool = False) -> str: - """Fetch Zenodo TIF, ingest to scratch, reproject to the final asset. - - Args: - gcp_project: GCP project for GCS + EE. - force: If True, delete existing scratch and final assets before run. - - Returns: - Final GEE asset ID. - """ - if force: - delete_asset_if_present(SCRATCH_ASSET_ID) - delete_asset_if_present(GEE_IPCC_CLIMATE_ZONES) - elif asset_exists(SCRATCH_ASSET_ID): - # A prior partial run left scratch behind but the final asset is - # missing (orchestrator only invoked us because of that). Clear - # scratch so the ingest can run cleanly. - logger.info('IPCC: clearing leftover scratch asset from prior run') - delete_asset_if_present(SCRATCH_ASSET_ID) - - run_id = uuid.uuid4().hex[:8] - work_dir = tempfile.mkdtemp(prefix=f'ipcc_climate_zones_{run_id}_') - - try: - local_tif = os.path.join(work_dir, SOURCE_FILENAME) - # Mirror-first fetch; Zenodo record probe is deferred to mirror miss. - fetch_with_mirror( - local_tif, - dataset='ipcc_climate_zones', - filename=SOURCE_FILENAME, - gcp_project=gcp_project, - source=lambda: _discover_zenodo_tif_url()[1], - timeout_s=300.0, - ) - - gcs_uri = upload_to_gcs( - gcp_project, GCS_BUCKET_NAME, GCS_STAGING_BLOB, local_tif - ) - try: - start_ingestion_and_wait( - gcs_uri, - SCRATCH_ASSET_ID, - band_name=BAND_NAME, - allow_overwrite=force, - ) - - export_task = _start_reproject_export( - SCRATCH_ASSET_ID, GEE_IPCC_CLIMATE_ZONES - ) - wait_for_export_task(export_task, GEE_IPCC_CLIMATE_ZONES) - finally: - delete_gcs_blob(gcp_project, GCS_BUCKET_NAME, GCS_STAGING_BLOB) - - # Final asset is written; scratch no longer needed. - delete_asset_if_present(SCRATCH_ASSET_ID) - finally: - shutil.rmtree(work_dir, ignore_errors=True) - - logger.info(f'IPCC: extract complete → {GEE_IPCC_CLIMATE_ZONES}') - return GEE_IPCC_CLIMATE_ZONES diff --git a/jdluc/extract/mirror.py b/jdluc/extract/mirror.py deleted file mode 100644 index 75e35ad..0000000 --- a/jdluc/extract/mirror.py +++ /dev/null @@ -1,136 +0,0 @@ -"""GCS mirror for non-native source-file fetches. - -Every upstream download in the extract modules flows through -``fetch_with_mirror``. The mirror lives at -``gs://{GCS_MIRROR_BUCKET}/{GCS_MIRROR_PREFIX}/{dataset}/{filename}`` -using the bucket's default storage class. Subsequent fetches read from -the mirror; upstream is only hit on mirror miss. - -Motivation: upstream publishers rotate files (NASS QuickStats supersedes -older releases when a new one ships; tile-server signed URLs rotate; -Figshare/Zenodo records occasionally move). With the mirror in place -the pipeline is insulated from upstream availability after a first -successful extract. - -Licensing: all four mirrored datasets permit redistribution with -attribution. - -- Harris AGB: CC-BY 4.0 (Harris et al. 2021, *Nature Climate Change*). -- Huang BGB: CC-BY 4.0 (Huang et al. 2021, Figshare deposit). -- NASS QuickStats: US-government public domain (no restrictions). -- IPCC climate zones: open Zenodo deposit (Ogle et al. 2006). - -Attributions are preserved in each per-dataset extract module's -docstring; this mirror helper does not embed license metadata on the -GCS blobs themselves. -""" - -import logging -from collections.abc import Callable -from typing import Union - -from google.cloud import storage - -from jdluc.utils.constants import GCS_BUCKET_NAME -from jdluc.utils.gee import download_with_retries, upload_to_gcs - -logger = logging.getLogger(__name__) - -# GCS mirror location for the 4 non-native extract inputs. -# ``GCS_MIRROR_BUCKET`` is sourced from ``utils.constants`` (edit it there to -# repoint at a different bucket); the prefix is bucket-internal and fixed. -# The legacy ``luc_high_res/...`` prefix is preserved so existing mirrored -# fetches continue to be cache hits across the cliq → jdluc extraction. -GCS_MIRROR_PREFIX: str = 'luc_high_res/extract_mirror' - -# Type alias for the source: either a literal URL or a zero-arg callable -# that resolves to one. Deferred resolution is the path IPCC takes — -# Zenodo's download URL requires a record-api probe to discover, and we -# want to skip that probe entirely on a mirror hit. -SourceUrl = Union[str, Callable[[], str]] - - -def _gcs_blob_exists(gcp_project: str, bucket_name: str, blob_name: str) -> bool: - """Return True if the blob is present in GCS, False otherwise.""" - client = storage.Client(project=gcp_project) - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_name) - return bool(blob.exists()) - - -def _download_from_gcs( - gcp_project: str, bucket_name: str, blob_name: str, dst_path: str -) -> None: - """Stream a GCS blob to ``dst_path``.""" - - client = storage.Client(project=gcp_project) - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_name) - blob.download_to_filename(dst_path) - - -def fetch_with_mirror( - dst_path: str, - *, - dataset: str, - filename: str, - gcp_project: str, - source: SourceUrl, - timeout_s: float = 600.0, - retries: int = 3, - backoff_s: float = 2.0, -) -> None: - """Fetch a source file, preferring the GCS mirror over upstream. - - Semantics: - 1. Probe ``gs://{GCS_MIRROR_BUCKET}/{blob_name}`` where ``blob_name`` - follows the ``{GCS_MIRROR_PREFIX}/{dataset}/{filename}`` layout. - 2. On mirror hit, stream the blob to ``dst_path`` (fast intra-Google - read; no egress cost when GEE and GCS sit in the same region). - Done — upstream is never contacted. - 3. On mirror miss, resolve ``source`` to a URL (calling it if it's - a callable so deferred-discovery sources like Zenodo only probe - on miss), run ``download_with_retries`` with the usual retry - policy, then upload ``dst_path`` back to the mirror at Archive - storage class. - - Args: - dst_path: Local filesystem path to materialize. - dataset: Dataset key (stable, short — e.g. ``'nass_yields'``, - ``'harris_agb'``). Used as the mirror-layout directory. - filename: Stable file basename for the mirror key. Does NOT have - to match the upstream filename — if upstream renames the - source file (e.g. Zenodo hash-suffixed URLs), the mirror - filename is what we probe on subsequent runs. - gcp_project: GCP project used for GCS reads/writes. - source: Either a literal URL or a zero-arg callable that returns - one. Deferred resolution is intended for cases where the - upstream URL requires a discovery probe we want to skip on - mirror hit. - timeout_s, retries, backoff_s: Passed through to - ``download_with_retries``. - - Raises: - Whatever ``download_with_retries`` raises on exhausted retries. - ``google.cloud.storage`` exceptions if the mirror is unreachable. - """ - blob_name = f'{GCS_MIRROR_PREFIX}/{dataset:s}/{filename:s}' - gcs_uri = f'gs://{GCS_BUCKET_NAME}/{blob_name}' - - if _gcs_blob_exists(gcp_project, GCS_BUCKET_NAME, blob_name): - logger.info(f'mirror[{dataset}]: cache hit, reading from {gcs_uri}') - _download_from_gcs(gcp_project, GCS_BUCKET_NAME, blob_name, dst_path) - return - - logger.info(f'mirror[{dataset}]: cache miss, fetching upstream') - source_url = source() if callable(source) else source - download_with_retries( - source_url, - dst_path, - timeout_s=timeout_s, - retries=retries, - backoff_s=backoff_s, - ) - - logger.info(f'mirror[{dataset}]: writing upstream copy to {gcs_uri}') - upload_to_gcs(gcp_project, GCS_BUCKET_NAME, blob_name, dst_path) diff --git a/jdluc/extract/nass_yields.py b/jdluc/extract/nass_yields.py deleted file mode 100644 index 014141f..0000000 --- a/jdluc/extract/nass_yields.py +++ /dev/null @@ -1,167 +0,0 @@ -"""USDA NASS QuickStats yield extract. - -Downloads the gzipped ``qs.crops_{RELEASE_DATE}.txt.gz`` TSV from NASS, -pre-filters to CORN / SOYBEANS / WHEAT YIELD rows, writes those rows to -a CSV, stages the CSV to GCS, and ingests it as a FeatureCollection -asset via ``ee.data.startTableIngestion``. The CSV has no geometry -columns, so GEE creates features with null geometry — fine because -``transform/summary_tables.py`` reads only ``feature['properties']`` -and never touches geometry. - -The semantic filters (``AGG_LEVEL_DESC == 'STATE'``, -``UNIT_DESC == 'BU / ACRE'``, year ∈ ``NASS_YIELD_YEARS``), the 4-year -arithmetic mean, and the bu/acre → kg/ha conversion all live at the -transform layer (``transform/summary_tables.py``). Keeping those out of -extract means a NASS_YIELD_YEARS bump doesn't force a re-extract. - -The earlier draft of this module shipped rows inline as -``ee.FeatureCollection([ee.Feature, ...])`` + ``Export.table.toAsset``. -That hit GEE's 10 MB per-request payload limit on full-archive runs -(≈1.5 M rows). The GCS-staged CSV path mirrors how the raster extractors -(Harris AGB, GFW Peatlands) stay cloud-native and is unbounded in size. -""" - -import gzip -import logging -import os -import shutil -import tempfile -import uuid - -import pandas as pd - -from jdluc.extract.mirror import fetch_with_mirror -from jdluc.utils.constants import ( - GCS_BUCKET_NAME, - GEE_NASS_YIELDS, - NASS_QUICKSTATS_RELEASE_DATE, - NASS_QUICKSTATS_URL_TEMPLATE, - NASS_TO_CROP_GROUP, -) -from jdluc.utils.gee import ( - delete_asset_if_present, - delete_gcs_blob, - start_table_ingestion_and_wait, - upload_to_gcs, -) - -logger = logging.getLogger(__name__) - -# Column set kept in the upload payload. Anything else from QuickStats -# is discarded client-side to keep the FeatureCollection small. -ROW_COLUMNS: list[str] = [ - 'state_fips', - 'state_name', - 'commodity_desc', - 'year', - 'value_bu_per_acre', - 'agg_level_desc', - 'unit_desc', -] - -# QuickStats value-column entries we cannot numerically coerce; rows with -# these get dropped (consistent with the legacy script's behavior). -_SUPPRESSED_VALUE_TOKENS: frozenset[str] = frozenset({'(D)', '(Z)', '(S)', '(NA)'}) - -# Commodity families we care about — methodology's three row crops. -_VALID_COMMODITIES: frozenset[str] = frozenset(NASS_TO_CROP_GROUP) - -# Chunk size for streaming the ~1.5 GB compressed TSV through pandas. -_TSV_CHUNK_ROWS: int = 100_000 - -GCS_STAGING_PREFIX: str = 'luc_high_res/staging/nass_yields' - - -def _download_and_parse(gzip_path: str, gcp_project: str) -> pd.DataFrame: - """Download the QuickStats TSV and return a pre-filtered DataFrame.""" - url = NASS_QUICKSTATS_URL_TEMPLATE.format(date=NASS_QUICKSTATS_RELEASE_DATE) - filename = f'qs.crops_{NASS_QUICKSTATS_RELEASE_DATE}.txt.gz' - fetch_with_mirror( - gzip_path, - dataset='nass_yields', - filename=filename, - gcp_project=gcp_project, - source=url, - timeout_s=1800.0, - ) - - frames: list[pd.DataFrame] = [] - logger.info(f'NASS: parsing TSV at {gzip_path}') - with gzip.open(gzip_path, 'rt', encoding='utf-8', errors='replace') as fh: - reader = pd.read_csv( - fh, - sep='\t', - dtype=str, - chunksize=_TSV_CHUNK_ROWS, - on_bad_lines='skip', - ) - for chunk in reader: - mask = (chunk['STATISTICCAT_DESC'] == 'YIELD') & ( - chunk['COMMODITY_DESC'].isin(_VALID_COMMODITIES) - ) - filtered = chunk.loc[mask] - if not filtered.empty: - frames.append(filtered) - - if not frames: - raise RuntimeError('NASS: no YIELD rows found after pre-filter') - - df = pd.concat(frames, ignore_index=True) - df = df[~df['VALUE'].str.strip().isin(_SUPPRESSED_VALUE_TOKENS)] - df['value_bu_per_acre'] = df['VALUE'].str.replace(',', '').astype(float) - df['state_fips'] = df['STATE_FIPS_CODE'].str.zfill(2) - df['year'] = df['YEAR'].astype(int) - df = df.rename( - columns={ - 'STATE_NAME': 'state_name', - 'COMMODITY_DESC': 'commodity_desc', - 'AGG_LEVEL_DESC': 'agg_level_desc', - 'UNIT_DESC': 'unit_desc', - } - ) - return df[ROW_COLUMNS].reset_index(drop=True) - - -def extract_nass_yields(gcp_project: str, force: bool = False) -> str: - """Download QuickStats, filter, then ingest as a CSV via GCS. - - Args: - gcp_project: GCP project for both the mirror read/write path - in ``fetch_with_mirror`` and the GCS staging upload that - backs ``ee.data.startTableIngestion``. - force: If True, delete the existing asset before re-ingesting. - - Returns: - GEE FeatureCollection asset ID. - """ - if force: - delete_asset_if_present(GEE_NASS_YIELDS) - - run_id = uuid.uuid4().hex[:8] - work_dir = tempfile.mkdtemp(prefix=f'nass_yields_{run_id}_') - gzip_path = os.path.join( - work_dir, f'qs.crops_{NASS_QUICKSTATS_RELEASE_DATE}.txt.gz' - ) - csv_path = os.path.join(work_dir, 'nass_yields_raw.csv') - blob_name = f'{GCS_STAGING_PREFIX}/{run_id}/nass_yields_raw.csv' - try: - df = _download_and_parse(gzip_path, gcp_project) - logger.info(f'NASS: writing {len(df)} raw rows to {csv_path} for table ingest') - # No geometry columns — GEE will produce null-geometry features. - # That's fine: transform/summary_tables.py reads only properties. - df.to_csv(csv_path, index=False) - - gcs_uri = upload_to_gcs(gcp_project, GCS_BUCKET_NAME, blob_name, csv_path) - try: - start_table_ingestion_and_wait( - gcs_uri, - GEE_NASS_YIELDS, - allow_overwrite=force, - ) - finally: - delete_gcs_blob(gcp_project, GCS_BUCKET_NAME, blob_name) - finally: - shutil.rmtree(work_dir, ignore_errors=True) - - logger.info(f'NASS: extract complete → {GEE_NASS_YIELDS}') - return GEE_NASS_YIELDS diff --git a/jdluc/gcs.py b/jdluc/gcs.py new file mode 100644 index 0000000..989c783 --- /dev/null +++ b/jdluc/gcs.py @@ -0,0 +1,215 @@ +import dataclasses +import functools +import hashlib +import inspect +import logging +import os +import typing +import urllib.parse + +import git +import google.cloud.storage +import pandas +import xarray + +logger = logging.getLogger(__name__) + + +def get_bucket_name_prefix_from_uri(uri: str) -> tuple[str, str]: + parsed = urllib.parse.urlparse(uri) + bucket_name = parsed.netloc + prefix = parsed.path.lstrip("/") + return bucket_name, prefix + + +def get_uri_from_bucket_name_prefix(bucket_name: str, prefix: str) -> str: + return f"gs://{bucket_name:s}/{prefix:s}" + + +@functools.cache +def get_gcs_client(gcp_project: str) -> google.cloud.storage.Client: + return google.cloud.storage.Client(project=gcp_project) + + +def gcs_blob_exists(gcp_project: str, remote_path: str) -> bool: + bucket_name, prefix = get_bucket_name_prefix_from_uri(uri=remote_path) + return ( + get_gcs_client(gcp_project=gcp_project) + .bucket(bucket_name=bucket_name) + .blob(blob_name=prefix) + .exists() + ) + + +def upload_local_path_to_gcs( + gcp_project: str, local_path: str, remote_path: str +) -> None: + logger.info( + f"Uploading from {local_path=:s} to {remote_path=:s} within {gcp_project=:s}" + ) + bucket_name, prefix = get_bucket_name_prefix_from_uri(uri=remote_path) + ( + get_gcs_client(gcp_project=gcp_project) + .bucket(bucket_name=bucket_name) + .blob(blob_name=prefix) + .upload_from_filename(filename=local_path) + ) + + +def write_dask_dataset_to_zarr(dset: xarray.Dataset, path_to_zarr: str) -> None: + assert dset.chunks is not None, f"{dset=:} is not a chunked dask array" + + import dask.config + import dask.diagnostics + import rasterio + + from jdluc.config import Config + + num_workers = Config.from_dot_env().number_of_dask_workers + + logger.info( + f"Saving to {path_to_zarr=:s} with {num_workers=:d} and {dset.chunksizes=:}" + ) + with ( + rasterio.Env( + GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", + CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".tif,.vrt", + ), + dask.config.set(scheduler="threads", num_workers=num_workers), + dask.diagnostics.ProgressBar(dt=5, minimum=1), + ): + dset.drop_vars("spatial_ref").to_zarr( + compute=False, consolidated=False, group=None, store=path_to_zarr + ).compute() + + +def open_zarr_to_dask_dataset(path_to_zarr: str) -> xarray.Dataset: + import rioxarray # noqa + + logger.info(f"Loading {path_to_zarr=:s}") + dset = xarray.open_zarr(store=path_to_zarr, consolidated=False) + assert dset.chunks is not None + assert isinstance(dset, xarray.Dataset) + assert dset.encoding["source"] == path_to_zarr + return dset.rio.write_crs(4326) + + +P = typing.ParamSpec("P") +R = typing.TypeVar("R") + + +class CacherProtocol(typing.Protocol[R]): + def __init__(self, *, bucket_name: str, hash_key: str) -> None: ... + @property + def exists_uri(self) -> str: ... + def deserialize(self) -> R: ... + def serialize(self, value: R) -> None: ... + + +@dataclasses.dataclass +class ParquetCacher: + bucket_name: str + hash_key: str + + @property + def gcs_uri(self) -> str: + return get_uri_from_bucket_name_prefix( + bucket_name=self.bucket_name, prefix=f"{self.hash_key:s}.parquet" + ) + + exists_uri = gcs_uri + + def deserialize(self) -> pandas.DataFrame: + logger.info(f"Loading from {self.gcs_uri=:s}") + return pandas.read_parquet(path=self.gcs_uri) + + def serialize(self, df: pandas.DataFrame) -> None: + logger.info(f"Saving to {self.gcs_uri=:s}") + df.to_parquet(path=self.gcs_uri) + + +@dataclasses.dataclass +class ZarrCacher: + bucket_name: str + hash_key: str + + @property + def gcs_uri(self) -> str: + return get_uri_from_bucket_name_prefix( + bucket_name=self.bucket_name, prefix=f"{self.hash_key:s}.zarr" + ) + + @property + def exists_uri(self) -> str: + return f"{self.gcs_uri:s}/zarr.json" + + def deserialize(self) -> xarray.Dataset: + return open_zarr_to_dask_dataset(path_to_zarr=self.gcs_uri) + + def serialize(self, dset: xarray.Dataset) -> None: + write_dask_dataset_to_zarr(dset=dset, path_to_zarr=self.gcs_uri) + + +RETURN_TYPE_TO_CACHE_CLS: dict[typing.Any, type[ParquetCacher] | type[ZarrCacher]] = { + pandas.DataFrame: ParquetCacher, + xarray.Dataset: ZarrCacher, +} + + +class CacherDecoratorProtocol(typing.Protocol): + def __call__( + self, func: typing.Callable[P, R] + ) -> functools._lru_cache_wrapper[R]: ... + + +def cache( + version: int, ignored_args: list[str] | None = None +) -> CacherDecoratorProtocol: + def get_module_for_func(func: typing.Callable[..., object]) -> str: + relative_path = os.path.relpath( + path=inspect.getfile(func), + start=git.Repo(search_parent_directories=True).working_dir, + ) + return relative_path.removesuffix(".py").replace(os.sep, ".") + + def decorator( + func: typing.Callable[P, R], + ) -> functools._lru_cache_wrapper[R]: + cache_cls = RETURN_TYPE_TO_CACHE_CLS[inspect.signature(func).return_annotation] + + from jdluc.config import Config + + @functools.cache + @functools.wraps(func) + def inner(*args: P.args, **kwargs: P.kwargs) -> R: + hash_tokens = ( + get_module_for_func(func=func), + func.__qualname__, + version, + *args, + *( + (arg, value) + for arg, value in sorted(kwargs.items()) + if (ignored_args is None) or (arg not in ignored_args) + ), + ) + data = "|".join(map(str, hash_tokens)).encode() + hash_key = hashlib.sha1(data=data).hexdigest()[:12] + config = Config.from_dot_env() + cacher = typing.cast( + CacherProtocol[R], + cache_cls(bucket_name=config.scratch_bucket_name, hash_key=hash_key), + ) + if gcs_blob_exists( + gcp_project=config.gcp_project, + remote_path=cacher.exists_uri, + ): + ret = cacher.deserialize() + else: + ret = func(*args, **kwargs) + cacher.serialize(ret) + return ret + + return inner + + return decorator diff --git a/jdluc/geo.py b/jdluc/geo.py new file mode 100644 index 0000000..1f77f0f --- /dev/null +++ b/jdluc/geo.py @@ -0,0 +1,141 @@ +import collections.abc +import logging +import math +import typing + +import geopandas +import numpy +import rasterio +import rio_cogeo.cogeo +import rio_cogeo.profiles +import shapely +import xarray + +logger = logging.getLogger(__name__) + + +def set_band_names_for_geotiff( + band_names: collections.abc.Sequence[str], path_to_geotiff: str +) -> None: + logger.info(f"Setting {band_names=:} for {path_to_geotiff=:s}") + with rasterio.open(fp=path_to_geotiff, mode="r+") as dataset: + for idx, band_name in enumerate(band_names, start=1): + dataset.set_band_description(idx, band_name) + + +def get_overview_level(height: int, width: int, minimum_pixels: int = 64) -> int: + if (pixel_ratio := min(height, width) // minimum_pixels) > 0: + return math.floor(math.log2(pixel_ratio)) + else: + return 0 + + +def convert_geotiff_to_cog( + metadata: dict[str, str], + no_data: float | int | None, + path_to_cog: str, + path_to_geotiff: str, +) -> None: + logger.info(f"Getting overview level from {path_to_geotiff=:s}") + with rasterio.open(fp=path_to_geotiff) as dataset: + overview_level = get_overview_level( + height=dataset.height, + width=dataset.width, + ) + logger.info(f"Converting from {path_to_geotiff=:s} to {path_to_cog=:s}") + rio_cogeo.cogeo.cog_translate( + additional_cog_metadata=metadata, + config={ + "GDAL_NUM_THREADS": "ALL_CPUS", + "GDAL_TIFF_INTERNAL_MASK": True, + }, + dst_kwargs=dict(rio_cogeo.profiles.DEFLATEProfile()) | {"BIGTIFF": "IF_SAFER"}, + dst_path=path_to_cog, + in_memory=False, + nodata=no_data, + overview_level=overview_level, + quiet=False, + source=path_to_geotiff, + ) + + +def convert_vector_to_flatgeobuf( + id_column_names: tuple[str, ...], + name_column_names: tuple[str, ...], + path_to_flatgeobuf: str, + path_to_vector: str, +) -> None: + logger.info( + f"Opening {path_to_vector=:s} for {id_column_names=:} and {name_column_names=:}" + ) + gdf: geopandas.GeoDataFrame = geopandas.read_file( + filename=path_to_vector, columns=id_column_names + name_column_names + ) + logger.info( + f"Forming {id_column_names=:} and {name_column_names=:}, dissolving, and saving to {path_to_flatgeobuf=:s}" + ) + ( + geopandas.GeoDataFrame( + crs=gdf.crs, + data={ + "id": gdf[list(id_column_names)].astype(str).agg(" | ".join, axis=1), + "name": gdf[list(name_column_names)] + .astype(str) + .agg(" | ".join, axis=1), + }, + geometry=gdf.geometry, + ) + .dissolve(by="id") + .reset_index(drop=False) + .to_file(path_to_flatgeobuf, driver="FlatGeobuf", SPATIAL_INDEX=True) + ) + + +def validate_geotiff(path_to_geotiff: str) -> None: + logger.info(f"Validating integrity of {path_to_geotiff=:s}") + with rasterio.open(path_to_geotiff) as dataset: + assert isinstance(dataset, rasterio.DatasetReader) + assert dataset.crs is not None + assert dataset.crs.to_epsg() == 4326 + assert dataset.height > 0 + assert dataset.width > 0 + assert dataset.transform.a > 0 + assert dataset.transform.e < 0 + assert dataset.width == dataset.height + + +def get_chunk_size( + dtypes: collections.abc.Iterable[numpy.dtype], + number_of_dimensions: int, + # 4 GiB + max_bytes_per_chunk: int = 1 << 32, +) -> int: + # Sum over all the variables + bytes_per_pixel = sum(dtype.itemsize for dtype in dtypes) + pixels_per_chunk = max_bytes_per_chunk / bytes_per_pixel + # Round down to nearest power of two + return 1 << int(math.log2(pixels_per_chunk) / number_of_dimensions) + + +ATTRS_TO_MOVE = ("_FillValue", "scale_factor", "add_offset", "dtype", "missing_value") + + +def unify_dtype_and_no_data(darray: xarray.DataArray) -> xarray.DataArray: + import rioxarray # noqa + + if (no_data := darray.rio.nodata) is not None: + darray = darray.where(darray != no_data, other=numpy.nan) + darray = darray.astype(numpy.float32).rio.write_nodata(numpy.nan) + assert isinstance(darray, xarray.DataArray) + for key in ATTRS_TO_MOVE: + if key in darray.attrs: + darray.encoding[key] = darray.attrs.pop(key) + return darray.transpose(..., "y", "x") + + +def clip_dset(dset: xarray.Dataset, geometry: shapely.Geometry) -> xarray.Dataset: + assert isinstance(geometry, shapely.Polygon | shapely.MultiPolygon) + import rioxarray # noqa + + ret = dset.rio.clip_box(*geometry.bounds).rio.clip([geometry], drop=True) + return typing.cast(xarray.Dataset, ret) diff --git a/jdluc/harmonize.py b/jdluc/harmonize.py new file mode 100644 index 0000000..209b071 --- /dev/null +++ b/jdluc/harmonize.py @@ -0,0 +1,394 @@ +"""Mosaic ingested source tiles onto a common grid and harmonize them into one dataset. + +For a tile set, each raster dataset's tiles are stitched into a per-band GDAL VRT, warped to +the shared GLAD 30m / 0.00025° grid (4,000 px/degree), and returned as an xarray.Dataset with +one variable per source band. The result is cached to GCS (`@gcs.cache`). + +Example invocation: + uv run python jdluc/harmonize.py CONUS +""" + +import argparse +import collections.abc +import dataclasses +import logging +import tempfile +import typing + +import numpy +import rasterio +import rasterio.enums +import rasterio.shutil +import rasterio.transform +import rasterio.vrt +import rioxarray +import xarray + +from jdluc import config, gcs, geo, ingest, tiling +from jdluc.datasets import NAME_TO_CLS, DatasetName, base + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class XY: + x: int + y: int + + def validated(self) -> typing.Self: + assert self.x >= 0 and self.y >= 0 + return self + + +RIO_TO_GDAL_DTYPE = { + "uint8": "Byte", + "uint16": "UInt16", + "int16": "Int16", + "uint32": "UInt32", + "int32": "Int32", + "float32": "Float32", + "float64": "Float64", +} + + +@dataclasses.dataclass +class Tile: + dtype: str + gcs_uri: str + no_data: float | int | None + resampling: rasterio.enums.Resampling + resolution: XY + + @classmethod + def from_dataset_tile_id( + cls, bucket_name: str, dataset: base.RasterDataset, tile_id: str + ) -> typing.Self: + gcs_uri = gcs.get_uri_from_bucket_name_prefix( + bucket_name=bucket_name, prefix=dataset.get_gcs_prefix(tile_id=tile_id) + ) + with rasterio.open(fp=gcs_uri) as ds: + rio_dtype = next(iter(ds.dtypes)) + return cls( + dtype=RIO_TO_GDAL_DTYPE[rio_dtype], + gcs_uri=gcs_uri, + no_data=ds.nodata, + resampling=dataset.resampling, + resolution=XY(x=ds.width, y=ds.height).validated(), + ) + + @property + def gdal_gcs_uri(self) -> str: + return "/vsigs/" + self.gcs_uri.removeprefix("gs://") + + +def iter_vrt_band_header( + band_name: str, dtype: str, no_data: int | float | None +) -> collections.abc.Iterator[str]: + yield f' ' + yield f" {band_name:s}" + yield f" {no_data}" + + +def iter_vrt_band_content( + band_idx: int, + dest_offset: XY, + dest_resolution: XY, + path_to_tile: str, + resampling: rasterio.enums.Resampling, + src_offset: XY, + src_resolution: XY, +) -> collections.abc.Iterator[str]: + yield f' ' + yield f' {path_to_tile:s}' + yield f" {band_idx:d}" + yield f' ' + yield f' ' + yield " " + + +@dataclasses.dataclass +class Grid: + origin: XY + tiles: XY + tile_resolution: XY + + @property + def epsg(self) -> int: + return 4326 + + @classmethod + def from_tile_ids_resolution( + cls, tile_ids: collections.abc.Sequence[str], tile_resolution: XY + ) -> typing.Self: + lats, lons = zip(*map(tiling.get_lat_lon_for_tile_id, tile_ids)) + return cls( + origin=XY(x=min(lons), y=max(lats)), + tiles=XY( + x=(max(lons) - min(lons)) // 10 + 1, + y=(max(lats) - min(lats)) // 10 + 1, + ).validated(), + tile_resolution=tile_resolution, + ) + + @property + def transform(self) -> tuple[float, float, float, float, float, float]: + return ( + self.origin.x, + 10 / self.tile_resolution.x, + 0, + self.origin.y, + 0, + -10 / self.tile_resolution.y, + ) + + @property + def resolution(self) -> XY: + return XY( + x=self.tiles.x * self.tile_resolution.x, + y=self.tiles.y * self.tile_resolution.y, + ).validated() + + @property + def iter_preamble(self) -> collections.abc.Iterator[str]: + yield f'' + yield f" EPSG:{self.epsg:d}" + yield f" {', '.join(map(str, self.transform))}" + + def get_offset_for_tile(self, tile_id: str) -> XY: + lat, lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) + return XY( + x=(lon - self.origin.x) // 10 * self.tile_resolution.x, + # Y index increases downward + y=(self.origin.y - lat) // 10 * self.tile_resolution.y, + ).validated() + + def get_offset_for_world(self, resolution: XY, span: XY) -> XY: + pixels_per_degree = XY( + x=resolution.x // span.x, + y=resolution.y // span.y, + ).validated() + return XY( + x=(self.origin.x + span.x // 2) * pixels_per_degree.x, + y=(span.y // 2 - self.origin.y) * pixels_per_degree.y, + ).validated() + + def get_resolution_for_world(self, resolution: XY, span: XY) -> XY: + return XY( + x=resolution.x * 10 // span.x * self.tiles.x, + y=resolution.y * 10 // span.y * self.tiles.y, + ).validated() + + +def get_vrt_for_dataset_band_tile_ids( + band_idx: int, + band_name: str, + bucket_name: str, + dataset: base.RasterDataset, + grid: Grid, + tile_ids: collections.abc.Sequence[str], +) -> str: + lines = list(grid.iter_preamble) + + logger.info(f"Processing {dataset=} and {band_name=:s}") + if dataset.partitioning == tiling.Partitioning.TEN_DEGREE_TILE: + for tile_idx, tile_id in enumerate(tile_ids): + tile = Tile.from_dataset_tile_id( + bucket_name=bucket_name, + dataset=dataset, + tile_id=tile_id, + ) + if tile_idx == 0: + lines.extend( + iter_vrt_band_header( + band_name=band_name, + dtype=tile.dtype, + no_data=tile.no_data, + ) + ) + lines.extend( + iter_vrt_band_content( + band_idx=band_idx, + dest_offset=grid.get_offset_for_tile(tile_id=tile_id), + dest_resolution=grid.tile_resolution, + path_to_tile=tile.gdal_gcs_uri, + resampling=tile.resampling, + src_offset=XY(x=0, y=0), + src_resolution=tile.resolution, + ) + ) + elif dataset.partitioning == tiling.Partitioning.WHOLE_WORLD: + (tile_id,) = tiling.NAME_TO_TILE_SET[tiling.TileSetName.WHOLE_WORLD] + tile = Tile.from_dataset_tile_id( + bucket_name=bucket_name, + dataset=dataset, + tile_id=tile_id, + ) + lines.extend( + iter_vrt_band_header( + band_name=band_name, + dtype=tile.dtype, + no_data=tile.no_data, + ) + ) + lines.extend( + iter_vrt_band_content( + band_idx=band_idx, + dest_offset=XY(x=0, y=0), + dest_resolution=grid.resolution, + path_to_tile=tile.gdal_gcs_uri, + resampling=tile.resampling, + src_offset=grid.get_offset_for_world( + resolution=tile.resolution, + span=XY(x=360, y=180), + ), + src_resolution=grid.get_resolution_for_world( + resolution=tile.resolution, + span=XY(x=360, y=180), + ), + ) + ) + else: + raise ValueError(dataset.partitioning) + lines.append(" ") + lines.append("") + + with tempfile.NamedTemporaryFile( + delete=False, mode="w", suffix=".mosaic.vrt" + ) as fp: + logger.info(f"Writing mosaic to {fp.name=:s}") + fp.writelines(line + "\n" for line in lines) + mosaic_path = fp.name + + with ( + rasterio.open(mosaic_path) as mosaic_fp, + rasterio.vrt.WarpedVRT( + mosaic_fp, + crs="EPSG:4326", + transform=rasterio.transform.Affine.from_gdal(*grid.transform), + width=grid.resolution.x, + height=grid.resolution.y, + resampling=rasterio.enums.Resampling.nearest, + ) as warped_vrt, + tempfile.NamedTemporaryFile( + delete=False, mode="w", suffix=".warped.vrt" + ) as warped_fp, + ): + logger.info(f"Writing warped VRT to {warped_fp.name=:s}") + rasterio.shutil.copy(warped_vrt, warped_fp.name, driver="VRT") + return warped_fp.name + + +def get_dset_for_output(path_to_vrts: collections.abc.Sequence[str]) -> xarray.Dataset: + darrays: list[xarray.DataArray] = [] + chunk_size = geo.get_chunk_size( + dtypes=[numpy.dtype("float32")] * len(path_to_vrts), number_of_dimensions=2 + ) + for path_to_vrt in path_to_vrts: + logger.info(f"Opening {path_to_vrt=:s} with {chunk_size=:d}") + darray = rioxarray.open_rasterio( + filename=path_to_vrt, + chunks=chunk_size, + # Remove the serialization lock because this is read-only + lock=False, + ) + assert isinstance(darray, xarray.DataArray) + darrays.append( + geo.unify_dtype_and_no_data( + darray=darray.isel(band=0) + .drop_vars("band") + .rename(darray.attrs.pop("long_name")) + ) + ) + + return xarray.Dataset({darray.name: darray for darray in darrays}) + + +GLAD_TILE_RESOLUTION = XY( + x=tiling.PIXELS_PER_TEN_DEGREE_TILE, y=tiling.PIXELS_PER_TEN_DEGREE_TILE +) + + +@gcs.cache(version=1) +def workflow( + dataset_names: tuple[DatasetName, ...], tile_ids: tuple[str, ...] +) -> xarray.Dataset: + logger.info( + f"Running the harmonize workflow for {dataset_names=:} and {tile_ids=:}" + ) + datasets = list(map(NAME_TO_CLS.__getitem__, dataset_names)) + cfg = config.Config.from_dot_env() + + for dataset in datasets: + tile_set = ( + ("world",) + if dataset.partitioning == tiling.Partitioning.WHOLE_WORLD + else tile_ids + ) + ingest.workflow( + bucket_name=cfg.ingest_bucket_name, + dataset=dataset, + concurrency=ingest.DEFAULT_CONCURRENCY, + gcp_project=cfg.gcp_project, + overwrite=False, + tile_set=tile_set, + ) + + logger.info("Constructing common grid") + grid = Grid.from_tile_ids_resolution( + tile_ids=tile_ids, tile_resolution=GLAD_TILE_RESOLUTION + ) + path_to_vrts = [ + get_vrt_for_dataset_band_tile_ids( + band_idx=band_idx, + band_name=band_name, + bucket_name=cfg.ingest_bucket_name, + dataset=dataset, + grid=grid, + tile_ids=tile_ids, + ) + for dataset in datasets + if isinstance(dataset, base.RasterDataset) + for band_idx, band_name in enumerate( + dataset.fully_qualified_band_names, start=1 + ) + ] + return get_dset_for_output(path_to_vrts=path_to_vrts) + + +DATASET_NAMES = ( + DatasetName.GFW_GLOBAL_PEATLANDS, + DatasetName.GFW_HARRIS_AGB, + DatasetName.GLAD_GLCLUC, + DatasetName.HUANG_BGB, + DatasetName.IPCC_CLIMATE_ZONES, + DatasetName.SOILGRIDS_OCS, + DatasetName.USDA_NASS_CDL, +) +assert set(DATASET_NAMES) == { + dataset_name + for dataset_name in DatasetName + if isinstance(NAME_TO_CLS[dataset_name], base.RasterDataset) +} + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "tile_set_name", choices=sorted(e.name for e in tiling.TileSetName) + ) + args = parser.parse_args() + + tile_set_name = tiling.TileSetName[str(args.tile_set_name)] + workflow( + dataset_names=DATASET_NAMES, tile_ids=tiling.NAME_TO_TILE_SET[tile_set_name] + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/jdluc/ingest.py b/jdluc/ingest.py new file mode 100644 index 0000000..d5745e7 --- /dev/null +++ b/jdluc/ingest.py @@ -0,0 +1,95 @@ +"""Ingest a named dataset over a named tile set. + +The parser is positional — `tile_set_name dataset_name` — plus optional `--concurrency` +and `--overwrite`. Tile sets come from `tiling.TileSetName`; datasets from +`datasets.DatasetName`. + +Example invocations: + uv run python -m jdluc.ingest WHOLE_WORLD IPCC_CLIMATE_ZONES + uv run python -m jdluc.ingest CONUS GLAD_GLCLUC + uv run python -m jdluc.ingest DELAWARE GFW_GLOBAL_PEATLANDS --concurrency=8 --overwrite +""" + +import argparse +import concurrent.futures +import logging + +from jdluc import config, datasets, tiling +from jdluc.datasets import base + + +def workflow( + bucket_name: str, + concurrency: int, + gcp_project: str, + overwrite: bool, + dataset: base.RasterDataset | base.TabularDataset | base.VectorDataset, + tile_set: tiling.TileSetType, +) -> dict[str, str | Exception]: + tile_id_is_valid_func = tiling.PARTITIONING_TO_IS_VALID_TILE_ID[ + dataset.partitioning + ] + if not all(map(tile_id_is_valid_func, tile_set)): + raise ValueError(f"{tile_set=:} are not valid for {dataset=:}") + results: dict[str, str | Exception] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures: dict[concurrent.futures.Future[str], str] = {} + for tile_id in tile_set: + futures[ + executor.submit( + dataset.ingest_a_tile, + bucket_name=bucket_name, + gcp_project=gcp_project, + overwrite=overwrite, + tile_id=tile_id, + ) + ] = tile_id + for future in concurrent.futures.as_completed(futures): + tile_id = futures[future] + try: + result = future.result() + except Exception as exc: + results[tile_id] = exc + else: + results[tile_id] = result + return results + + +DEFAULT_CONCURRENCY = 4 + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "tile_set_name", choices=sorted(e.name for e in tiling.TileSetName) + ) + parser.add_argument( + "dataset_name", choices=sorted(e.name for e in datasets.DatasetName) + ) + parser.add_argument("--concurrency", default=DEFAULT_CONCURRENCY, type=int) + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + dataset_name = datasets.DatasetName[str(args.dataset_name)] + tile_set_name = tiling.TileSetName[str(args.tile_set_name)] + cfg = config.Config.from_dot_env() + results: dict[str, str | Exception] = workflow( + bucket_name=cfg.ingest_bucket_name, + concurrency=int(args.concurrency), + dataset=datasets.NAME_TO_CLS[dataset_name], + gcp_project=cfg.gcp_project, + overwrite=args.overwrite, + tile_set=tiling.NAME_TO_TILE_SET[tile_set_name], + ) + for tile_id, result in results.items(): + print(tile_id, repr(result)) + return sum(1 for result in results.values() if isinstance(result, Exception)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/jdluc/pipeline.py b/jdluc/pipeline.py deleted file mode 100644 index 96f4f10..0000000 --- a/jdluc/pipeline.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Pipeline orchestration for jdLUC land use change emissions. - -Provides ``run_pipeline()`` which drives the Extract -> Transform -> Publish -stages in sequence, returning a ``PipelineResult`` with per-stage asset IDs. - -See specs/pipeline_tech_design.md § pipeline.py for details. -""" - -import dataclasses -import logging - -from jdluc.extract.extract import ( - ExtractError, - ExtractResult, - extract_all, -) -from jdluc.publish.publish import PublishResult, run_publish, target_entries -from jdluc.transform.transform import run_transform -from jdluc.utils.gee import initialize_gee - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass -class PipelineResult: - """Result of a full pipeline run.""" - - version: str - region_name: str - states: list[str] - extract_result: ExtractResult | None = None - land_use_asset_id: str | None = None - emissions_asset_id: str | None = None - transitions_table_id: str | None = None - crops_table_id: str | None = None - publish_result: PublishResult | None = None - from_cache: bool = False - - -def run_pipeline( - gcp_project: str, - states: list[str], - region_name: str, - force: bool = False, -) -> PipelineResult: - """Run the full jdLUC pipeline: extract -> transform -> publish. - - Args: - gcp_project: GCP project ID for Earth Engine. - states: List of state FIPS codes. - region_name: Region label for asset naming (e.g. 'delaware'). - force: If True, re-export even if cached assets exist. - - Returns: - PipelineResult with version, region, asset IDs, and per-stage - results. ``from_cache`` is True iff every stage was a cache hit - (transform `from_cache` AND every BQ target's `from_cache`). - - Raises: - ExtractError: If any dataset in ``extract_all`` failed — transform - is not attempted so a broken upstream dataset doesn't produce - silently-wrong downstream outputs. - PublishError: If any publish target failed. - """ - logger.info(f'Initializing Earth Engine with project: {gcp_project}') - initialize_gee(gcp_project) - - # Extract - extract_result = extract_all(gcp_project=gcp_project, force=force) - if extract_result.failed: - raise ExtractError(extract_result) - - # Transform - transform_result = run_transform( - gcp_project=gcp_project, - states=states, - region_name=region_name, - force=force, - ) - - # Publish — only valid when transform produced both tables (single-state - # and multi-state both populate transitions_table_id + crops_table_id; - # cached-only re-runs do too). - if ( - transform_result.transitions_table_id is None - or transform_result.crops_table_id is None - or transform_result.land_use_asset_id is None - or transform_result.emissions_asset_id is None - ): - raise RuntimeError( - 'Transform completed without all required asset IDs ' - '(transitions_table_id, crops_table_id, land_use_asset_id, ' - 'emissions_asset_id) — publish stage cannot proceed.' - ) - - publish_result = run_publish( - region_name=region_name, - transitions_asset_id=transform_result.transitions_table_id, - crops_asset_id=transform_result.crops_table_id, - land_use_asset_id=transform_result.land_use_asset_id, - emissions_asset_id=transform_result.emissions_asset_id, - transform_version=transform_result.version, - force=force, - ) - - publish_from_cache = all( - target.from_cache for _, target in target_entries(publish_result) - ) - - return PipelineResult( - version=transform_result.version, - region_name=region_name, - states=states, - extract_result=extract_result, - land_use_asset_id=transform_result.land_use_asset_id, - emissions_asset_id=transform_result.emissions_asset_id, - transitions_table_id=transform_result.transitions_table_id, - crops_table_id=transform_result.crops_table_id, - publish_result=publish_result, - from_cache=transform_result.from_cache and publish_from_cache, - ) diff --git a/jdluc/publish/__init__.py b/jdluc/publish/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jdluc/publish/bigquery.py b/jdluc/publish/bigquery.py deleted file mode 100644 index 897f754..0000000 --- a/jdluc/publish/bigquery.py +++ /dev/null @@ -1,260 +0,0 @@ -"""BigQuery publish target. - -Exports the regional ``transitions`` and ``crops`` GEE table assets -produced by the transform stage into BigQuery for downstream SQL -queries. Both tables land in the same ``{BQ_PROJECT}.{BQ_DATASET}`` and -carry compound-keyed names of the form -``{prefix}_{region}_{transform_sha}_{publish_sha}`` — the name alone is -the cache key, so a content change in either dimension produces a new -table and no silent overwrites are possible. - -Architecture: -- One shared primitive (``export_feature_collection_to_bigquery``) - carries the real export + cache-probe logic. -- Two thin per-table wrappers (``export_transitions_to_bigquery``, - ``export_crops_to_bigquery``) exist for call-site clarity and to - anchor per-table schema documentation. - -See specs/pipeline_tech_design.md § Publish. -""" - -import logging - -import ee - -from jdluc.publish.publish import BigQueryExportResult -from jdluc.utils.asset_management import ( - parse_compound_version_from_asset_id, -) -from jdluc.utils.constants import ( - BQ_CROPS_TABLE_PREFIX, - BQ_DATASET, - BQ_JOB_LOCATION, - BQ_PROJECT, - BQ_TRANSITIONS_TABLE_PREFIX, -) -from jdluc.utils.gee import wait_for_export_task - -# [[NOTE: ``BQ_JOB_LOCATION`` in utils/constants.py pins the dataset -# location for documentation/future use. ``ee.batch.Export.table.toBigQuery`` -# does not accept an explicit location — it infers from the target -# project.dataset pair — so the constant is not referenced at the call -# site here. If we ever switch to a ``google-cloud-bigquery`` load-job -# pattern, the constant becomes load-bearing and gets passed through.]] - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Table-ID builders -# --------------------------------------------------------------------------- - - -def build_transitions_bq_table_id( - region_name: str, - transform_version: str, - publish_version: str, -) -> str: - """Compound-keyed BQ table ID for the ``transitions`` target. - - Schema: ``TRANSITIONS_TABLE_COLUMNS`` — county_fips × epoch_transition × - emissions_type grain. - """ - return _build_table_id( - BQ_TRANSITIONS_TABLE_PREFIX, region_name, transform_version, publish_version - ) - - -def build_crops_bq_table_id( - region_name: str, - transform_version: str, - publish_version: str, -) -> str: - """Compound-keyed BQ table ID for the ``crops`` target. - - Schema: ``CROPS_TABLE_COLUMNS`` — county_fips × crop_code grain, with - emissions factors, yields, and driver-breakdown columns attached. - """ - return _build_table_id( - BQ_CROPS_TABLE_PREFIX, region_name, transform_version, publish_version - ) - - -def _build_table_id( - prefix: str, - region_name: str, - transform_version: str, - publish_version: str, -) -> str: - table_short = f'{prefix}_{region_name}_{transform_version}_{publish_version}' - return f'{BQ_PROJECT}.{BQ_DATASET}.{table_short}' - - -# --------------------------------------------------------------------------- -# Cache probe -# --------------------------------------------------------------------------- - - -def bq_table_exists(table_id: str) -> bool: - """Return True iff the BQ table exists. - - Uses ``google-cloud-bigquery.Client().get_table``; a ``NotFound`` - exception (google-api-core) resolves to False. Lazy imports the BQ - client so modules that don't need a cache probe (e.g. the `publish.py` - import) don't pay for it. - """ - # Lazy imports: the BQ client is a heavy transitive dep. - from google.api_core import exceptions as gcs_exceptions - from google.cloud import bigquery - - client = bigquery.Client(project=BQ_PROJECT) - try: - client.get_table(table_id) - except gcs_exceptions.NotFound: - return False - return True - - -def ensure_bq_dataset_exists() -> None: - """Create ``{BQ_PROJECT}.{BQ_DATASET}`` if it doesn't already exist. - - The first publish on a fresh project would otherwise fail with - ``Not found: Dataset`` from ``Export.table.toBigQuery`` — auto-creating - here removes a manual prereq step from pipeline bring-up. Idempotent: - a 409 (already-exists, including races between parallel runs) is - treated as success. - - Raises whatever ``google-cloud-bigquery`` raises if the caller lacks - BigQuery dataset-create permission — fail-fast is correct since the - ``Export.table.toBigQuery`` calls would also fail. - """ - from google.api_core import exceptions as gcs_exceptions - from google.cloud import bigquery - - client = bigquery.Client(project=BQ_PROJECT) - dataset_id = f'{BQ_PROJECT}.{BQ_DATASET}' - try: - client.get_dataset(dataset_id) - return - except gcs_exceptions.NotFound: - pass - - logger.info(f'Creating BQ dataset {dataset_id} (location={BQ_JOB_LOCATION})') - dataset = bigquery.Dataset(dataset_id) - dataset.location = BQ_JOB_LOCATION - try: - client.create_dataset(dataset) - except gcs_exceptions.Conflict: - # Race: another process beat us to it. Treat as success. - pass - - -# --------------------------------------------------------------------------- -# Export primitives -# --------------------------------------------------------------------------- - - -def export_feature_collection_to_bigquery( - asset_id: str, - table_id: str, - force: bool = False, -) -> BigQueryExportResult: - """Shared export primitive for any GEE ``FeatureCollection`` asset. - - Decision tree: - - ``force=False`` AND ``bq_table_exists(table_id)`` → skip; return a - ``BigQueryExportResult`` with ``from_cache=True``. - - Otherwise → submit ``ee.batch.Export.table.toBigQuery`` with - ``overwrite=True`` and block on ``wait_for_export_task``. - - Hard failures in the export task are captured on the returned result's - ``error`` field rather than raised — ``run_publish`` decides whether - to raise ``PublishError``. - """ - versions = parse_compound_version_from_asset_id(table_id) - if versions is None: - raise ValueError( - f'table_id {table_id!r} does not end in a compound ' - '{transform_sha}_{publish_sha} suffix — build it via ' - 'build_transitions_bq_table_id / build_crops_bq_table_id.' - ) - transform_version, publish_version = versions - - if not force and bq_table_exists(table_id): - logger.info(f'BQ cache hit: {table_id}') - return BigQueryExportResult( - table_id=table_id, - transform_version=transform_version, - publish_version=publish_version, - from_cache=True, - ) - - ensure_bq_dataset_exists() - logger.info(f'BQ export: {asset_id} -> {table_id}') - try: - task = ee.batch.Export.table.toBigQuery( - collection=ee.FeatureCollection(asset_id), - description=_export_description(table_id), - table=table_id, - overwrite=True, - append=False, - ) - task.start() - wait_for_export_task(task, table_id) - except Exception as exc: - logger.error(f'BQ export failed for {table_id}: {exc}') - return BigQueryExportResult( - table_id=table_id, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - error=str(exc), - ) - - return BigQueryExportResult( - table_id=table_id, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - ) - - -def _export_description(table_id: str) -> str: - """GEE export-task description string. Truncated to stay under the 100-char cap.""" - short = table_id.split('.', 2)[-1] - return f'publish_{short}'[:100] - - -# --------------------------------------------------------------------------- -# Per-table wrappers -# --------------------------------------------------------------------------- - - -def export_transitions_to_bigquery( - transitions_asset_id: str, - table_id: str, - force: bool = False, -) -> BigQueryExportResult: - """Export the regional ``transitions`` table to BigQuery. - - ``transitions_asset_id`` must point at a GEE FeatureCollection with - ``TRANSITIONS_TABLE_COLUMNS`` — the (county_fips, epoch_transition, - emissions_type) grain the transform stage emits. - """ - return export_feature_collection_to_bigquery( - transitions_asset_id, table_id, force=force - ) - - -def export_crops_to_bigquery( - crops_asset_id: str, - table_id: str, - force: bool = False, -) -> BigQueryExportResult: - """Export the regional ``crops`` table to BigQuery. - - ``crops_asset_id`` must point at a GEE FeatureCollection with - ``CROPS_TABLE_COLUMNS`` — the (county_fips, crop_code) grain with - emissions factors, yields, and driver breakdown columns attached. - """ - return export_feature_collection_to_bigquery(crops_asset_id, table_id, force=force) diff --git a/jdluc/publish/bq_to_gcs.py b/jdluc/publish/bq_to_gcs.py deleted file mode 100644 index 32c8a87..0000000 --- a/jdluc/publish/bq_to_gcs.py +++ /dev/null @@ -1,83 +0,0 @@ -"""BQ-to-GCS publish target. - -Exports the regional ``transitions`` and ``crops`` BigQuery tables to GCS -as CSV files. BQ may shard large tables into multiple files; the wildcard -destination URI (``{name}-*.csv``) handles that transparently. - -Architecture mirrors ``bigquery.py``: -- One shared primitive (``export_bq_table_to_gcs``) carries the extract-job - and cache-probe logic. -- Two thin per-table wrappers exist for call-site clarity. -""" - -import logging - -from jdluc.publish.publish import GCSExportResult -from jdluc.utils.asset_management import parse_compound_version_from_asset_id -from jdluc.utils.constants import BQ_PROJECT, GCS_BUCKET_NAME, GCS_TABLE_PREFIX - -logger = logging.getLogger(__name__) - - -def build_table_gcs_uri(table_id: str) -> str: - _, _, table_name = table_id.rpartition('.') - return f'gs://{GCS_BUCKET_NAME}/{GCS_TABLE_PREFIX}/{table_name}-*.csv' - - -def gcs_csv_exists(gcs_uri: str) -> bool: - from google.cloud import storage - - # Strip 'gs://{bucket}/' and the trailing wildcard to get a list prefix. - prefix = gcs_uri.removeprefix(f'gs://{GCS_BUCKET_NAME}/').replace('*', '') - client = storage.Client() - blobs = client.list_blobs(GCS_BUCKET_NAME, prefix=prefix, max_results=1) - return any(True for _ in blobs) - - -def export_bq_table_to_gcs(table_id: str, force: bool = False) -> GCSExportResult: - versions = parse_compound_version_from_asset_id(table_id) - if versions is None: - raise ValueError( - f'table_id {table_id!r} does not end in a compound ' - '{transform_sha}_{publish_sha} suffix.' - ) - transform_version, publish_version = versions - gcs_uri = build_table_gcs_uri(table_id) - - if not force and gcs_csv_exists(gcs_uri): - logger.info(f'BQ->GCS cache hit: {gcs_uri}') - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=True, - ) - - logger.info(f'BQ->GCS export: {table_id} -> {gcs_uri}') - try: - from google.cloud import bigquery - - job = bigquery.Client(project=BQ_PROJECT).extract_table( - table_id, - gcs_uri, - job_config=bigquery.ExtractJobConfig( - destination_format=bigquery.DestinationFormat.CSV, - ), - ) - job.result() - except Exception as exc: - logger.error(f'BQ->GCS export failed for {table_id}: {exc}') - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - error=str(exc), - ) - - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - ) diff --git a/jdluc/publish/gcs.py b/jdluc/publish/gcs.py deleted file mode 100644 index 71f573e..0000000 --- a/jdluc/publish/gcs.py +++ /dev/null @@ -1,182 +0,0 @@ -"""GCS publish target. - -Exports the regional ``land_use`` and ``emissions`` GEE ``ee.Image`` assets -produced by the transform stage into GCS as cloud-optimised GeoTIFFs. -Objects land at ``gs://{GCS_BUCKET_NAME}/{GCS_RASTER_PREFIX}/{name}_{region}_{t_sha}_{p_sha}``. -GEE may write a single ``{prefix}.tif`` (single tile) or -``{prefix}-00000-of-NNNNN.tif`` (multi-tile) depending on the image footprint. - -Architecture mirrors ``bigquery.py``: -- One shared primitive (``export_image_to_gcs``) carries the real export + - cache-probe logic. -- Two thin per-asset wrappers (``export_land_use_to_gcs``, - ``export_emissions_to_gcs``) exist for call-site clarity. - -See specs/pipeline_tech_design.md § Publish. -""" - -import logging - -import ee - -from jdluc.publish.publish import GCSExportResult -from jdluc.utils.constants import GCS_BUCKET_NAME, GCS_RASTER_PREFIX -from jdluc.utils.gee import wait_for_export_task - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# GCS path builder -# --------------------------------------------------------------------------- - - -def build_raster_gcs_prefix( - name: str, - region_name: str, - transform_version: str, - publish_version: str, -) -> str: - """GCS object prefix (without bucket) for a named raster export.""" - return f'{GCS_RASTER_PREFIX}/{name}_{region_name}_{transform_version}_{publish_version}' - - -# --------------------------------------------------------------------------- -# Cache probe -# --------------------------------------------------------------------------- - - -def gcs_raster_exists(prefix: str) -> bool: - """Return True iff any GCS object matching ``prefix`` already exists. - - Uses ``list_blobs(prefix=prefix, max_results=1)`` — if any blob matches - the prefix the export is already present. Lazy import, same posture as - the BQ probe in ``bigquery.py``. - """ - from google.cloud import storage - - client = storage.Client() - blobs = client.list_blobs(GCS_BUCKET_NAME, prefix=prefix, max_results=1) - return any(True for _ in blobs) - - -# --------------------------------------------------------------------------- -# Export primitive -# --------------------------------------------------------------------------- - - -def export_image_to_gcs( - asset_id: str, - gcs_prefix: str, - name: str, - region_name: str, - transform_version: str, - publish_version: str, - force: bool = False, -) -> GCSExportResult: - """Shared export primitive for any GEE ``ee.Image`` asset. - - Decision tree: - - ``force=False`` AND ``gcs_raster_exists(gcs_prefix)`` → skip; return - a ``GCSExportResult`` with ``from_cache=True``. - - Otherwise → submit ``ee.batch.Export.image.toCloudStorage`` and block - on ``wait_for_export_task``. - - Hard failures are captured on the returned result's ``error`` field - rather than raised — ``run_publish`` decides whether to raise - ``PublishError``. - """ - gcs_uri = f'gs://{GCS_BUCKET_NAME}/{gcs_prefix}' - - if not force and gcs_raster_exists(gcs_prefix): - logger.info(f'GCS cache hit: {gcs_uri}') - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=True, - ) - - logger.info(f'GCS export: {asset_id} -> {gcs_uri}') - try: - img = ee.Image(asset_id) - projection = img.projection().getInfo() - description = f'publish_{name}_{region_name}'[:100] - - task = ee.batch.Export.image.toCloudStorage( - image=img, - description=description, - bucket=GCS_BUCKET_NAME, - fileNamePrefix=gcs_prefix, - crs=projection['crs'], - crsTransform=projection['transform'], - region=img.geometry(), - fileFormat='GeoTIFF', - maxPixels=int(1e13), - formatOptions={'cloudOptimized': True}, - ) - task.start() - wait_for_export_task(task, gcs_uri) - except Exception as exc: - logger.error(f'GCS export failed for {gcs_uri}: {exc}') - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - error=str(exc), - ) - - return GCSExportResult( - gcs_uri=gcs_uri, - transform_version=transform_version, - publish_version=publish_version, - from_cache=False, - ) - - -# --------------------------------------------------------------------------- -# Per-asset wrappers -# --------------------------------------------------------------------------- - - -def export_land_use_to_gcs( - asset_id: str, - gcs_prefix: str, - region_name: str, - transform_version: str, - publish_version: str, - *, - force: bool = False, -) -> GCSExportResult: - """Export the regional ``land_use`` raster to GCS as a cloud-optimised GeoTIFF.""" - return export_image_to_gcs( - asset_id, - gcs_prefix, - name='land_use', - region_name=region_name, - transform_version=transform_version, - publish_version=publish_version, - force=force, - ) - - -def export_emissions_to_gcs( - asset_id: str, - gcs_prefix: str, - region_name: str, - transform_version: str, - publish_version: str, - *, - force: bool = False, -) -> GCSExportResult: - """Export the regional ``emissions`` raster to GCS as a cloud-optimised GeoTIFF.""" - return export_image_to_gcs( - asset_id, - gcs_prefix, - name='emissions', - region_name=region_name, - transform_version=transform_version, - publish_version=publish_version, - force=force, - ) diff --git a/jdluc/publish/publish.py b/jdluc/publish/publish.py deleted file mode 100644 index 25efec8..0000000 --- a/jdluc/publish/publish.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Publish stage entry point. - -Orchestrates publish targets. Today the only target is BigQuery (two -tables: ``transitions`` and ``crops``); the dataclass surface is -target-keyed so future publish targets (tile-serving, GCS exports, -report generation) land as additional fields on ``PublishResult`` -without reshaping ``pipeline.py``. - -See specs/pipeline_tech_design.md § Publish for the full design. -""" - -import dataclasses -import logging -from concurrent.futures import ThreadPoolExecutor - -from jdluc.utils.version import compute_publish_version - -# GCSExportResult is defined here (not in gcs.py) so publish.py stays the -# single home for all publish-stage result types and gcs.py can import it -# without a circular dependency. - - -@dataclasses.dataclass -class GCSExportResult: - """Outcome of a single GCS raster export. - - - ``gcs_uri`` — ``gs://{bucket}/{prefix}`` (no ``.tif`` suffix). - - ``transform_version`` / ``publish_version`` — provenance. - - ``from_cache`` — True iff blobs matching the prefix already existed. - - ``error`` — non-None if the export failed. - """ - - gcs_uri: str - transform_version: str - publish_version: str - from_cache: bool - error: str | None = None - - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass -class BigQueryExportResult: - """Outcome of a single BigQuery export. - - - ``table_id`` — fully qualified BQ table ID - (``{project}.{dataset}.{prefix}_{region}_{t_sha}_{p_sha}``). - - ``transform_version`` / ``publish_version`` — parsed back out of - ``table_id`` for provenance. Kept as explicit fields so callers - don't have to re-parse the name themselves. - - ``from_cache`` — True iff the table already existed at the expected - name and the export was skipped. - - ``error`` — non-None if the export failed. ``run_publish`` raises - ``PublishError`` iff any target's ``error`` is non-None. - """ - - table_id: str - transform_version: str - publish_version: str - from_cache: bool - error: str | None = None - - -@dataclasses.dataclass -class PublishResult: - """Aggregated outcome of a ``run_publish`` invocation. - - One field per publish target. - """ - - transitions: BigQueryExportResult - crops: BigQueryExportResult - land_use: GCSExportResult - emissions: GCSExportResult - transitions_csv: GCSExportResult - crops_csv: GCSExportResult - - -class PublishError(Exception): - """Raised by ``run_publish`` when any target's ``error`` field is set. - - Carries the full ``PublishResult`` so callers can see which targets - succeeded (cached or freshly exported) and which failed, with the - per-target error messages. Analogous to ``ExtractError`` and - ``TransformError``. - """ - - def __init__(self, result: PublishResult) -> None: - self.result = result - failed = [ - (name, target.error) - for name, target in target_entries(result) - if target.error is not None - ] - failed_summary = ', '.join(f'{name}({msg})' for name, msg in failed) - super().__init__( - f'Publish failed for {len(failed)} target(s): {failed_summary}' - ) - - -def target_entries( - result: PublishResult, -) -> list[tuple[str, BigQueryExportResult | GCSExportResult]]: - """Enumerate the per-target fields on a ``PublishResult``. - - Kept as a module-level helper (not a method on ``PublishResult``) so - ``PublishError`` and any future iterator needs — for example the - ``cli.py`` output block — have one place to update - when targets are added. Order is stable: matches the dataclass - field order. - """ - return [ - ('transitions', result.transitions), - ('crops', result.crops), - ('land_use', result.land_use), - ('emissions', result.emissions), - ('transitions_csv', result.transitions_csv), - ('crops_csv', result.crops_csv), - ] - - -def run_publish( - region_name: str, - transitions_asset_id: str, - crops_asset_id: str, - land_use_asset_id: str, - emissions_asset_id: str, - transform_version: str, - force: bool = False, -) -> PublishResult: - """Publish-stage entry point. Exports all four targets in parallel. - - Args: - region_name: Region label; embedded in table/object names. - transitions_asset_id: GEE FeatureCollection asset ID for the - regional ``transitions`` table. - crops_asset_id: GEE FeatureCollection asset ID for the regional - ``crops`` table. - land_use_asset_id: GEE Image asset ID for the regional - ``land_use`` raster. - emissions_asset_id: GEE Image asset ID for the regional - ``emissions`` raster. - transform_version: SHA carried on the input GEE assets; embedded - in published artifact names. - force: If True, re-export all targets ignoring caches. - - Returns: - ``PublishResult`` with one result per target. - - Raises: - ``PublishError``: iff any target's ``error`` field is non-None. - All four targets attempt regardless of each other's outcome, - so a ``PublishResult`` is always produced before this raise. - """ - publish_version = compute_publish_version() - logger.info( - f'Publish: region={region_name} transform={transform_version} ' - f'publish={publish_version} force={force}' - ) - - # Lazy imports keep ``publish/publish.py`` cheap to import (no - # ``ee`` / ``google-cloud-bigquery`` / ``google-cloud-storage`` cost - # when only the dataclasses are needed, e.g. by ``pipeline.py``). - from jdluc.publish.bigquery import ( - build_crops_bq_table_id, - build_transitions_bq_table_id, - export_crops_to_bigquery, - export_transitions_to_bigquery, - ) - from jdluc.publish.bq_to_gcs import export_bq_table_to_gcs - from jdluc.publish.gcs import ( - build_raster_gcs_prefix, - export_emissions_to_gcs, - export_land_use_to_gcs, - ) - - transitions_table_id = build_transitions_bq_table_id( - region_name, transform_version, publish_version - ) - crops_table_id = build_crops_bq_table_id( - region_name, transform_version, publish_version - ) - land_use_prefix = build_raster_gcs_prefix( - 'land_use', region_name, transform_version, publish_version - ) - emissions_prefix = build_raster_gcs_prefix( - 'emissions', region_name, transform_version, publish_version - ) - - # All four targets attempt regardless of each other's outcome (each - # captures its own errors on the returned result), so running them in - # parallel reduces wall-clock for non-cached publish runs. - with ThreadPoolExecutor(max_workers=6) as ex: - transitions_future = ex.submit( - export_transitions_to_bigquery, - transitions_asset_id, - transitions_table_id, - force=force, - ) - crops_future = ex.submit( - export_crops_to_bigquery, crops_asset_id, crops_table_id, force=force - ) - land_use_future = ex.submit( - export_land_use_to_gcs, - land_use_asset_id, - land_use_prefix, - region_name, - transform_version, - publish_version, - force=force, - ) - emissions_future = ex.submit( - export_emissions_to_gcs, - emissions_asset_id, - emissions_prefix, - region_name, - transform_version, - publish_version, - force=force, - ) - - # Resolve BQ futures first so tables exist before phase 2 submits. - transitions_result = transitions_future.result() - crops_result = crops_future.result() - - transitions_csv_future = ex.submit( - export_bq_table_to_gcs, transitions_table_id, force=force - ) - crops_csv_future = ex.submit( - export_bq_table_to_gcs, crops_table_id, force=force - ) - - land_use_result = land_use_future.result() - emissions_result = emissions_future.result() - transitions_csv_result = transitions_csv_future.result() - crops_csv_result = crops_csv_future.result() - - result = PublishResult( - transitions=transitions_result, - crops=crops_result, - land_use=land_use_result, - emissions=emissions_result, - transitions_csv=transitions_csv_result, - crops_csv=crops_csv_result, - ) - - if any(target.error is not None for _, target in target_entries(result)): - raise PublishError(result) - - logger.info( - f'Publish: transitions {"cached" if transitions_result.from_cache else "exported"} ' - f'-> {transitions_result.table_id}' - ) - logger.info( - f'Publish: crops {"cached" if crops_result.from_cache else "exported"} ' - f'-> {crops_result.table_id}' - ) - logger.info( - f'Publish: land_use {"cached" if land_use_result.from_cache else "exported"} ' - f'-> {land_use_result.gcs_uri}' - ) - logger.info( - f'Publish: emissions {"cached" if emissions_result.from_cache else "exported"} ' - f'-> {emissions_result.gcs_uri}' - ) - logger.info( - f'Publish: transitions_csv {"cached" if transitions_csv_result.from_cache else "exported"} ' - f'-> {transitions_csv_result.gcs_uri}' - ) - logger.info( - f'Publish: crops_csv {"cached" if crops_csv_result.from_cache else "exported"} ' - f'-> {crops_csv_result.gcs_uri}' - ) - return result diff --git a/jdluc/tiling.py b/jdluc/tiling.py new file mode 100644 index 0000000..16fd874 --- /dev/null +++ b/jdluc/tiling.py @@ -0,0 +1,455 @@ +import collections.abc +import enum +import itertools +import logging +import math +import operator +import re +import typing + +import networkx +import shapely + +logger = logging.getLogger(__name__) + +PIXELS_PER_TEN_DEGREE_TILE = 40_000 + + +class Partitioning(enum.Enum): + @typing.override + def __str__(self) -> str: + return self.name.lower().replace("_", "-") + + ONE_DEGREE_TILE = enum.auto() + TEN_DEGREE_TILE = enum.auto() + WHOLE_WORLD = enum.auto() + XYZ_MERCATOR_TILE = enum.auto() + + +PARTITIONING_TO_IS_VALID_TILE_ID: dict[ + Partitioning, collections.abc.Callable[[str], bool] +] = { + # one- and ten-degree validations are the same + Partitioning.ONE_DEGREE_TILE: ( + lambda tile_id: ( + re.compile(r"\d\d[NS]_\d\d\d[EW]").fullmatch(string=tile_id) is not None + ) + ), + Partitioning.TEN_DEGREE_TILE: ( + lambda tile_id: ( + re.compile(r"\d0[NS]_\d\d0[EW]").fullmatch(string=tile_id) is not None + ) + ), + Partitioning.WHOLE_WORLD: (lambda tile_id: tile_id == "world"), + Partitioning.XYZ_MERCATOR_TILE: ( + lambda tile_id: ( + re.compile(r"\d\d\dX_\d\d\dY_\d\dZ").fullmatch(string=tile_id) is not None + ) + ), +} +assert set(Partitioning) == set(PARTITIONING_TO_IS_VALID_TILE_ID), ( + f"{set(Partitioning)=:}; {set(PARTITIONING_TO_IS_VALID_TILE_ID)=:}" +) + + +def get_lat_lon_for_tile_id(tile_id: str) -> tuple[int, int]: + lat_str, lon_str = tile_id.split("_") + return ( + int(lat_str[:2]) * (+1 if lat_str[-1] == "N" else -1), + int(lon_str[:3]) * (+1 if lon_str[-1] == "E" else -1), + ) + + +def get_tile_id_for_lat_lon(lat: int, lon: int) -> str: + return f"{abs(lat):02d}{'N' if lat >= 0 else 'S'}_{abs(lon):03d}{'E' if lon >= 0 else 'W'}" + + +def iter_ten_degree_tile_id_for_geometry( + geometry: shapely.Polygon | shapely.MultiPolygon, +) -> collections.abc.Iterator[str]: + def get_range_for_low_high(low: float, high: float) -> range: + return range( + math.floor(low / 10) * 10, + math.ceil(high / 10) * 10, + 10, + ) + + min_lon, min_lat, max_lon, max_lat = geometry.bounds + + for lat in get_range_for_low_high(low=min_lat, high=max_lat): + for lon in get_range_for_low_high(low=min_lon, high=max_lon): + tile = shapely.box(xmin=lon, ymin=lat, xmax=lon + 10, ymax=lat + 10) + if tile.intersects(other=geometry): + # NB: convention is to use the northern lat for the id + yield get_tile_id_for_lat_lon(lat=lat + 10, lon=lon) + + +class TileSetName(enum.Enum): + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: list[str] + ) -> str: + return name + + BAY_AREA = enum.auto() + CONUS = enum.auto() + DELAWARE = enum.auto() + GFW = enum.auto() + WHOLE_WORLD = enum.auto() + + +TileSetType = tuple[str, ...] +NAME_TO_TILE_SET: dict[TileSetName, TileSetType] = { + TileSetName.CONUS: ( + "30N_080W", + "30N_090W", + "30N_100W", + "30N_110W", + "30N_120W", + "40N_070W", + "40N_080W", + "40N_090W", + "40N_100W", + "40N_110W", + "40N_120W", + "40N_130W", + "50N_070W", + "50N_080W", + "50N_090W", + "50N_100W", + "50N_110W", + "50N_120W", + "50N_130W", + ), + TileSetName.GFW: ( + "00N_000E", + "00N_010E", + "00N_020E", + "00N_030E", + "00N_040E", + "00N_040W", + "00N_050W", + "00N_060W", + "00N_070E", + "00N_070W", + "00N_080W", + "00N_090E", + "00N_090W", + "00N_100E", + "00N_100W", + "00N_110E", + "00N_120E", + "00N_130E", + "00N_140E", + "00N_150E", + "00N_160E", + "10N_000E", + "10N_010E", + "10N_010W", + "10N_020E", + "10N_020W", + "10N_030E", + "10N_040E", + "10N_050E", + "10N_050W", + "10N_060W", + "10N_070E", + "10N_070W", + "10N_080E", + "10N_080W", + "10N_090E", + "10N_090W", + "10N_100E", + "10N_100W", + "10N_110E", + "10N_120E", + "10N_130E", + "10S_010E", + "10S_020E", + "10S_030E", + "10S_040E", + "10S_040W", + "10S_050E", + "10S_050W", + "10S_060W", + "10S_070W", + "10S_080W", + "10S_110E", + "10S_120E", + "10S_130E", + "10S_140E", + "10S_150E", + "10S_160E", + "10S_170E", + "10S_180W", + "20N_000E", + "20N_010E", + "20N_010W", + "20N_020E", + "20N_020W", + "20N_030E", + "20N_040E", + "20N_050E", + "20N_060W", + "20N_070E", + "20N_070W", + "20N_080E", + "20N_080W", + "20N_090E", + "20N_090W", + "20N_100E", + "20N_100W", + "20N_110E", + "20N_110W", + "20N_120E", + "20N_120W", + "20N_160W", + "20S_010E", + "20S_020E", + "20S_030E", + "20S_040E", + "20S_050E", + "20S_050W", + "20S_060W", + "20S_070W", + "20S_080W", + "20S_110E", + "20S_120E", + "20S_130E", + "20S_140E", + "20S_150E", + "20S_160E", + "30N_000E", + "30N_010E", + "30N_010W", + "30N_020E", + "30N_020W", + "30N_030E", + "30N_040E", + "30N_050E", + "30N_060E", + "30N_070E", + "30N_080E", + "30N_080W", + "30N_090E", + "30N_090W", + "30N_100E", + "30N_100W", + "30N_110E", + "30N_110W", + "30N_120E", + "30N_120W", + "30N_160W", + "30N_170W", + "30S_010E", + "30S_020E", + "30S_030E", + "30S_060W", + "30S_070W", + "30S_080W", + "30S_110E", + "30S_120E", + "30S_130E", + "30S_140E", + "30S_150E", + "30S_170E", + "40N_000E", + "40N_010E", + "40N_010W", + "40N_020E", + "40N_020W", + "40N_030E", + "40N_040E", + "40N_050E", + "40N_060E", + "40N_070E", + "40N_070W", + "40N_080E", + "40N_080W", + "40N_090E", + "40N_090W", + "40N_100E", + "40N_100W", + "40N_110E", + "40N_110W", + "40N_120E", + "40N_120W", + "40N_130E", + "40N_130W", + "40N_140E", + "40S_070W", + "40S_080W", + "40S_140E", + "40S_160E", + "40S_170E", + "50N_000E", + "50N_010E", + "50N_010W", + "50N_020E", + "50N_030E", + "50N_040E", + "50N_050E", + "50N_060E", + "50N_060W", + "50N_070E", + "50N_070W", + "50N_080E", + "50N_080W", + "50N_090E", + "50N_090W", + "50N_100E", + "50N_100W", + "50N_110E", + "50N_110W", + "50N_120E", + "50N_120W", + "50N_130E", + "50N_130W", + "50N_140E", + "50N_150E", + "50S_060W", + "50S_070W", + "50S_080W", + "60N_000E", + "60N_010E", + "60N_010W", + "60N_020E", + "60N_020W", + "60N_030E", + "60N_040E", + "60N_050E", + "60N_060E", + "60N_060W", + "60N_070E", + "60N_070W", + "60N_080E", + "60N_080W", + "60N_090E", + "60N_090W", + "60N_100E", + "60N_100W", + "60N_110E", + "60N_110W", + "60N_120E", + "60N_120W", + "60N_130E", + "60N_130W", + "60N_140E", + "60N_140W", + "60N_150E", + "60N_150W", + "60N_160E", + "60N_160W", + "60N_170E", + "60N_170W", + "60N_180W", + "70N_000E", + "70N_010E", + "70N_020E", + "70N_020W", + "70N_030E", + "70N_030W", + "70N_040E", + "70N_050E", + "70N_060E", + "70N_070E", + "70N_070W", + "70N_080E", + "70N_080W", + "70N_090E", + "70N_090W", + "70N_100E", + "70N_100W", + "70N_110E", + "70N_110W", + "70N_120E", + "70N_120W", + "70N_130E", + "70N_130W", + "70N_140E", + "70N_140W", + "70N_150E", + "70N_150W", + "70N_160E", + "70N_160W", + "70N_170E", + "70N_170W", + "70N_180W", + "80N_010E", + "80N_020E", + "80N_030E", + "80N_050E", + "80N_060E", + "80N_070E", + "80N_070W", + "80N_080E", + "80N_080W", + "80N_090E", + "80N_090W", + "80N_100E", + "80N_100W", + "80N_110E", + "80N_110W", + "80N_120E", + "80N_120W", + "80N_130E", + "80N_130W", + "80N_140E", + "80N_140W", + "80N_150E", + "80N_150W", + "80N_160E", + "80N_160W", + "80N_170E", + "80N_170W", + ), + TileSetName.BAY_AREA: ("40N_130W",), + TileSetName.DELAWARE: ("40N_080W",), + TileSetName.WHOLE_WORLD: ("world",), +} +assert set(TileSetName) == set(NAME_TO_TILE_SET), ( + f"{set(TileSetName)=:}; {set(NAME_TO_TILE_SET)=:}" +) +is_sorted = lambda values: list(values) == sorted(values) +assert all(map(is_sorted, NAME_TO_TILE_SET.values())) + + +def tiles_are_adjacent(left: str, right: str) -> bool: + lats, lons = zip(*map(get_lat_lon_for_tile_id, (left, right))) + dlat, dlon = itertools.starmap(operator.sub, (lats, lons)) + if dlat == 0: + return abs(dlon) in {10, 350} + elif dlon == 0: + return abs(dlat) == 10 + else: + return False + + +EDGE_EXCLUSIONS = { + # Sever Alaska from CONUS + ("50N_130W", "60N_130W"), +} + + +def get_tile_clusters( + tile_ids: set[str], + is_adjacent: typing.Callable[[str, str], bool] = tiles_are_adjacent, + edge_exclusions: set[tuple[str, str]] = EDGE_EXCLUSIONS, +) -> list[set[str]]: + graph: networkx.Graph[str] = networkx.Graph() + + for tile_id in tile_ids: + # Add self-edge so solo tiles are not dropped + graph.add_edge(tile_id, tile_id) + for other in tile_ids: + if is_adjacent(tile_id, other): + if (tile_id, other) in edge_exclusions or ( + other, + tile_id, + ) in edge_exclusions: + logger.info( + "Skipping edge because it appears in the exclusion list" + ) + else: + graph.add_edge(tile_id, other) + return list(map(set, networkx.connected_components(graph))) diff --git a/jdluc/transform/__init__.py b/jdluc/transform/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jdluc/transform/_export.py b/jdluc/transform/_export.py deleted file mode 100644 index 1f41037..0000000 --- a/jdluc/transform/_export.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Shared image-export helper for the transform stage. - -`land_use.py` and `emissions.py` produce ~95% identical -``Export.image.toAsset`` shapes. This module owns the canonical export -shape so future changes to projection pinning, mask application, or -cache-skip semantics live in one place. -""" - -import logging -from typing import Literal - -import ee - -from jdluc.utils._ee_types import EEGeometry, EEImage -from jdluc.utils.asset_management import delete_asset_safely -from jdluc.utils.constants import ( - GLAD_CRS, - GLAD_CRS_TRANSFORM, - output_asset_id, -) - -logger = logging.getLogger(__name__) - - -def export_image_asset_to_gee( - *, - image: EEImage, - asset_kind: Literal['land_use', 'emissions'], - region: str, - version: str, - geometry: EEGeometry, - fips_mask: EEImage, - force: bool, -) -> ee.batch.Task | None: - """Export an image asset with the canonical jdLUC export shape. - - The export uses ``region=geometry.bounds()`` (a 4-vertex rectangle) - rather than the polygon itself, avoiding polygon-vertex-per-tile - overhead. The asset's polygon-shape masking is preserved by - ``image.updateMask(fips_mask)`` immediately before the export — - ``fips_mask`` is the run-scoped county-FIPS mask (1 inside the - requested states' CONUS counties, masked elsewhere), sourced from - the canonical ``GEE_COUNTY_FIPS_LABEL`` asset. - - ``crs`` and ``crsTransform`` are pinned on the export per the - GEE-canonical "pull-through" pattern: pinning at the output node - propagates back through the lazy graph, so each input is - materialized in the GLAD target grid exactly once at the export - node. Eager ``.reproject()`` calls on the inputs are deliberately - omitted — they trigger per-tile memory pressure (see the GEE - Projections guide's "Use reproject() with caution!" note). - - Cache semantics: probes via ``ee.data.getAsset`` when ``force`` is - False; on a hit, returns None and emits a ``cached, skipping export`` - log line. When ``force`` is True, deletes the existing asset before - submitting the new task. Returns the running task handle on a fresh - submission. - """ - asset_id = output_asset_id(asset_kind, region, version) - - if not force: - try: - ee.data.getAsset(asset_id) - logger.info(f'{asset_kind} asset cached, skipping export: {asset_id}') - return None - except ee.EEException as e: - if 'does not exist' not in str(e): - raise - else: - delete_asset_safely(asset_id) - - masked_image = image.updateMask(fips_mask) - task = ee.batch.Export.image.toAsset( - image=masked_image, - assetId=asset_id, - description=asset_id.rsplit('/', 1)[-1], - region=geometry.bounds(), - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e13), - ) - task.start() - logger.info(f'Started {asset_kind} export: {asset_id}') - return task diff --git a/jdluc/transform/emissions.py b/jdluc/transform/emissions.py deleted file mode 100644 index 2c81376..0000000 --- a/jdluc/transform/emissions.py +++ /dev/null @@ -1,484 +0,0 @@ -"""Per-state emissions raster construction. - -Builds the 11-band emissions_{region}_{version} raster from a previously -materialized land_use asset. - -See specs/pipeline_tech_design.md § emissions.py for details. -""" - -import logging -from dataclasses import dataclass - -import ee - -from jdluc.transform._export import export_image_asset_to_gee -from jdluc.transform.land_use import LAND_USE_BAND_NAMES -from jdluc.utils._ee_types import EEGeometry, EEImage -from jdluc.utils.constants import ( - CARBON_FRACTION_DEAD_WOOD, - CARBON_FRACTION_LITTER, - CARBON_FRACTION_LIVE, - CO2_C_RATIO, - DOM_FACTORS_BY_ZONE, - EMISSIONS_BAND_NAMES, - EMISSIVE_ENCODED_CODES, - GEE_HARRIS_AGB, - GEE_HUANG_BGB, - GEE_IPCC_CLIMATE_ZONES, - GEE_SOILGRIDS_SOC, - GHGP_EPOCH_WEIGHTS_2020, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - GLAD_EPOCH_PAIRS, - GRASSLAND_VEGETATION_TC_HA_BY_ZONE, - IPCC_CLIMATE_ZONE_NATIVE_REMAP, - IPCC_REFERENCE_SOC_STOCK_TC_HA, - IPCC_SOC_LOSS_FRACTIONS, - LAND_USE_CATEGORIES, - PEATLAND_DRAINAGE_ENCODED_CODES, - PEATLAND_E_LM_TCO2E_HA_YR, - PEATLAND_P_LUC_TCO2_HA, - ROOT_SHOOT_RATIO_TEMPERATE, - luc_emissions_band, - peatland_conversion_band, - transitions_band, -) -from jdluc.utils.transitions import decode_from_image, decode_to_image - -logger = logging.getLogger(__name__) - - -@dataclass -class EmissionsInputs: - """Pre-loaded GEE inputs to the per-state emissions graph. - - Every field is an `EEImage` already clipped to the state geometry and - pinned to the GLAD grid. Populated once by `build_emissions_image` and - threaded through all three `calculate_*` helpers so the graph-building - code doesn't need to re-load inputs per epoch. - """ - - agb: EEImage - bgb: EEImage - soc: EEImage - climate_zone: EEImage - is_peatland: EEImage - crops_2020: EEImage - pixel_area_ha: EEImage - - -# --------------------------------------------------------------------------- -# Per-pixel factor lookup tables (precomputed at module load) -# --------------------------------------------------------------------------- - -# DOM carbon fraction per IPCC climate zone: dw_factor × C_dw + litter × C_lit. -# Applied per-pixel via `climate_zone.remap(_DOM_KEYS, _DOM_VALUES)` to build -# the dom_factor_per_pixel image. -_DOM_KEYS: list[int] = list(DOM_FACTORS_BY_ZONE.keys()) -_DOM_VALUES: list[float] = [ - dw * CARBON_FRACTION_DEAD_WOOD + lit * CARBON_FRACTION_LITTER - for (dw, lit) in DOM_FACTORS_BY_ZONE.values() -] - -# Houghton/BLUE grassland total-vegetation-C (tC/ha), keyed by climate zone. -_GRASSLAND_KEYS: list[int] = list(GRASSLAND_VEGETATION_TC_HA_BY_ZONE.keys()) -_GRASSLAND_VALUES: list[float] = list(GRASSLAND_VEGETATION_TC_HA_BY_ZONE.values()) - -# SOC loss-fraction table, flattened into parallel key/value lists for a -# single `.remap()` call per epoch. Composite key = zone*10000 + from*100 + to -# (all three codes fit in two decimal digits). -_SOC_KEYS: list[int] = [ - zone * 10000 + from_code * 100 + to_code - for (zone, from_code, to_code) in IPCC_SOC_LOSS_FRACTIONS -] -_SOC_VALUES: list[float] = list(IPCC_SOC_LOSS_FRACTIONS.values()) - - -# --------------------------------------------------------------------------- -# Input loaders -# --------------------------------------------------------------------------- - - -def _load_agb(state_geometry: EEGeometry) -> EEImage: - """Harris et al. (2021) above-ground biomass, Mg biomass/ha. - - Tiles are uploaded pre-aligned to the GLAD grid, but - ``ImageCollection.mosaic()`` drops the per-tile projection metadata - and returns a degenerate ``scale=111319 m`` default. ``setDefaultProjection`` - re-attaches the GLAD projection without forcing a resample (the tiles - already live on that grid). Critical at multi-state scope: without - this, downstream operations fall back to the 1°/EPSG:4326 default - inherited from the missing-projection mosaic, which forces redundant - reprojection of every other input that touches the same expression. - """ - return ( - ee.ImageCollection(GEE_HARRIS_AGB) - .mosaic() - .setDefaultProjection(crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM) - .clip(state_geometry) - .select([0]) - .toFloat() - .rename('agb') - ) - - -def _load_bgb(state_geometry: EEGeometry, agb_image: EEImage) -> EEImage: - """Huang et al. (2021) below-ground biomass, Mg biomass/ha. - - Huang ships at ~1 km on a non-GLAD grid; ``.resample('bilinear')`` sets - the resampling kernel for continuous data, and the actual reprojection - onto ``GLAD_CRS_TRANSFORM`` happens once at the export node via the - Export call's pinned ``crs`` / ``crsTransform`` (the GEE pull-through - pattern). Calling ``.reproject()`` here forces eager materialization of - the entire reprojected raster at union scope, which trips - ``"User memory limit exceeded"`` on the diagnostic path and per-tile - memory pressure on the export path. NoData pixels are filled via IPCC - Tier 1 temperate R:S applied to the AGB input. - """ - huang = ( - ee.Image(GEE_HUANG_BGB).clip(state_geometry).select([0]).resample('bilinear') - ) - fallback = agb_image.multiply(ROOT_SHOOT_RATIO_TEMPERATE) - return huang.unmask(fallback).toFloat().rename('bgb') - - -def _load_soc(state_geometry: EEGeometry) -> EEImage: - """SoilGrids SOC stock (0–30 cm), tC/ha. - - Native 250 m Interrupted Goode Homolosine grid. ``.resample('bilinear')`` - sets the kernel for continuous-data resampling; the actual reprojection - onto the GLAD grid happens at the export node, not here (see ``_load_bgb`` - for the rationale). NoData pixels (<1% of conversion area) are gap-filled - with the IPCC reference SOC stock for warm-temperate-moist mineral soil - (IPCC 2019 Vol 4 Table 2.3). - """ - return ( - ee.Image(GEE_SOILGRIDS_SOC) - .clip(state_geometry) - .select([0]) - .resample('bilinear') - .unmask(IPCC_REFERENCE_SOC_STOCK_TC_HA) - .toFloat() - .rename('soc') - ) - - -def _load_ipcc_climate(state_geometry: EEGeometry) -> EEImage: - """IPCC climate-zone raster, remapped to canonical 1–10 codes. - - The asset is pre-aligned to the GLAD grid, so no reprojection. - Native Ogle codes are remapped to our canonical vocabulary via - IPCC_CLIMATE_ZONE_NATIVE_REMAP; pixels outside the vocabulary (polar - zones 11–12) fall through to 0. - """ - return ( - ee.Image(GEE_IPCC_CLIMATE_ZONES) - .clip(state_geometry) - .select([0]) - .remap( - list(IPCC_CLIMATE_ZONE_NATIVE_REMAP.keys()), - list(IPCC_CLIMATE_ZONE_NATIVE_REMAP.values()), - defaultValue=0, - ) - .rename('climate_zone') - ) - - -def _load_pixel_area_ha() -> EEImage: - """Per-pixel area in hectares. - - `ee.Image.pixelArea()` is a virtual image — area is computed on demand - at whatever output projection the consumer requests. Returning it - un-reprojected lets the export's pinned ``crs`` / ``crsTransform`` - materialize area at the GLAD grid exactly once at the output node; - eagerly ``.reproject()``-ing here forces the full GLAD-extent area - raster to be computed up front (1.3 B float32 cells at multi-state - scope), which is the textbook "request all inputs at very small scale - over a wide spatial extent" anti-pattern from the Projections guide. - """ - return ee.Image.pixelArea().divide(1e4).rename('area_ha') - - -def _load_land_use_bands(land_use_asset_id: str, state_geometry: EEGeometry) -> EEImage: - """Load the six-band cached land_use asset, clipped to the state. - - Single source of truth for the four `transitions_{epoch}` bands, - `crops_2020`, and `is_peatland` — everything in emissions.py that reads - the land_use side of the pipeline goes through this loader. - """ - return ee.Image(land_use_asset_id).clip(state_geometry).select(LAND_USE_BAND_NAMES) - - -# --------------------------------------------------------------------------- -# Per-epoch emissions calculation -# --------------------------------------------------------------------------- - - -def calculate_epoch_emissions( - encoded_transition: EEImage, - inputs: EmissionsInputs, - epoch: tuple[int, int], -) -> EEImage: - """Compute LUC + peatland conversion emissions for a single epoch. - - Returns a two-band float32 image `(luc_emissions_{epoch}, - peatland_conversion_{epoch})`. Vegetation and SOC paths are gated on the - 9-pair LUC-emissive mask; peatland conversion is gated on the narrower - 8-pair peatland-drainage mask (forest→short_veg excluded — vegetation - disturbance alone does not drain a peatland). - """ - from_year, to_year = epoch - from_code = decode_from_image(encoded_transition) - to_code = decode_to_image(encoded_transition) - - # Per-pixel factor images keyed by the pixel's IPCC climate zone. - dom_factor = inputs.climate_zone.remap( - _DOM_KEYS, _DOM_VALUES, defaultValue=0 - ).toFloat() - grassland_c = inputs.climate_zone.remap( - _GRASSLAND_KEYS, _GRASSLAND_VALUES, defaultValue=0 - ).toFloat() - - # Per-pixel forest vegetation-C stock: AGB_C + BGB_C + DOM_C, tC/ha. - forest_c = ( - inputs.agb.multiply(CARBON_FRACTION_LIVE) - .add(inputs.bgb.multiply(CARBON_FRACTION_LIVE)) - .add(inputs.agb.multiply(dom_factor)) - ) - - forest = LAND_USE_CATEGORIES['forest'].code - wetland_forest = LAND_USE_CATEGORIES['wetland_forest'].code - short_veg = LAND_USE_CATEGORIES['short_vegetation'].code - wetland_short_veg = LAND_USE_CATEGORIES['wetland_short_vegetation'].code - - # stock_from: per-pixel source-side vegetation C. 0 for any source - # category other than the four in the emissive vocabulary. - stock_from = ( - ee.Image.constant(0) - .toFloat() - .where(from_code.eq(forest), forest_c) - .where(from_code.eq(wetland_forest), forest_c) - .where(from_code.eq(short_veg), grassland_c) - .where(from_code.eq(wetland_short_veg), grassland_c) - ) - # stock_to: per-pixel destination-side vegetation C. Only short-veg - # destinations re-accumulate grassland stock; cropland/built_up/water/ - # snow_ice/bare destinations all retain 0. - stock_to = ( - ee.Image.constant(0) - .toFloat() - .where(to_code.eq(short_veg), grassland_c) - .where(to_code.eq(wetland_short_veg), grassland_c) - ) - - # Emissive-transition masks. `is_emissive` gates the vegetation + SOC - # paths (9-pair vocabulary); `is_peatland_drainage` gates peatland - # conversion (8-pair subset — cropland/built_up destinations only). - is_emissive = encoded_transition.remap( - EMISSIVE_ENCODED_CODES, - [1] * len(EMISSIVE_ENCODED_CODES), - defaultValue=0, - ) - is_peatland_drainage = encoded_transition.remap( - PEATLAND_DRAINAGE_ENCODED_CODES, - [1] * len(PEATLAND_DRAINAGE_ENCODED_CODES), - defaultValue=0, - ) - - # Vegetation emissions: clamp negative stock deltas to 0 so inert or - # stock-gaining transitions don't introduce removals. - veg_tco2 = ( - stock_from.subtract(stock_to) - .max(0) - .multiply(CO2_C_RATIO) - .multiply(inputs.pixel_area_ha) - .multiply(is_emissive) - ) - - # SOC loss fraction via composite-key .remap(), zero for non-emissive - # pairs by construction. - soc_key = ( - inputs.climate_zone.multiply(10000).add(from_code.multiply(100)).add(to_code) - ) - loss_fraction = soc_key.remap(_SOC_KEYS, _SOC_VALUES, defaultValue=0).toFloat() - soc_tco2 = ( - inputs.soc.multiply(loss_fraction) - .multiply(CO2_C_RATIO) - .multiply(inputs.pixel_area_ha) - .multiply(is_emissive) - ) - - luc_emissions = ( - veg_tco2.add(soc_tco2) - .unmask(0) - .toFloat() - .rename(luc_emissions_band(from_year, to_year)) - ) - peatland_conversion = ( - inputs.is_peatland.multiply(is_peatland_drainage) - .multiply(PEATLAND_P_LUC_TCO2_HA) - .multiply(inputs.pixel_area_ha) - .unmask(0) - .toFloat() - .rename(peatland_conversion_band(from_year, to_year)) - ) - return luc_emissions.addBands(peatland_conversion) - - -# --------------------------------------------------------------------------- -# Peatland occupation (annual, undiscounted) -# --------------------------------------------------------------------------- - - -def calculate_peatland_occupation(inputs: EmissionsInputs) -> EEImage: - """Compute the 2020 peatland-occupation emissions band. - - `E_LM × pixel_area_ha` applied on pixels that are both peatland and - classified as cropland in GLAD 2020 (encoded as "CDL crop code present" - in the land_use asset's `crops_2020` band). Masking via multiplication - rather than `.updateMask()` keeps the output a fully-populated float32 - grid with 0.0 outside the mask (Design decision 6). - """ - is_cropland_2020 = inputs.crops_2020.gt(0) - mask = inputs.is_peatland.multiply(is_cropland_2020) - return ( - mask.multiply(PEATLAND_E_LM_TCO2E_HA_YR) - .multiply(inputs.pixel_area_ha) - .unmask(0) - .toFloat() - .rename('peatland_occupation_2020') - ) - - -# --------------------------------------------------------------------------- -# 2020-epoch GHGP allocation -# --------------------------------------------------------------------------- - - -def calculate_allocated_emissions_2020( - luc_epoch_bands: list[EEImage], - peatland_epoch_bands: list[EEImage], - occupation_band: EEImage, -) -> EEImage: - """Compute the two 2020-allocated emissions bands. - - Per-epoch LUC and peatland-conversion emissions are discounted by the - GHGP 2020-epoch weights; peatland occupation is added at full weight - to the peatland-allocated band (it's an annual land-management emission, - not a conversion pulse). - """ - if len(luc_epoch_bands) != len(GLAD_EPOCH_PAIRS) or len( - peatland_epoch_bands - ) != len(GLAD_EPOCH_PAIRS): - raise ValueError( - 'expected one band per GLAD_EPOCH_PAIRS entry; got ' - f'{len(luc_epoch_bands)} LUC, {len(peatland_epoch_bands)} peatland' - ) - - allocated_luc = ee.Image.constant(0).toFloat() - allocated_peat = ee.Image.constant(0).toFloat() - for epoch, luc, peat in zip( - GLAD_EPOCH_PAIRS, luc_epoch_bands, peatland_epoch_bands - ): - weight = GHGP_EPOCH_WEIGHTS_2020[epoch] - allocated_luc = allocated_luc.add(luc.multiply(weight)) - allocated_peat = allocated_peat.add(peat.multiply(weight)) - allocated_peat = allocated_peat.add(occupation_band) - - return ( - allocated_luc.toFloat() - .rename('allocated_luc_emissions_2020') - .addBands(allocated_peat.toFloat().rename('allocated_peatland_emissions_2020')) - ) - - -# --------------------------------------------------------------------------- -# Main graph builder -# --------------------------------------------------------------------------- - - -def build_emissions_image(land_use_asset_id: str, geometry: EEGeometry) -> EEImage: - """Construct the 11-band emissions raster for the requested region. - - Reads the cached land_use asset (via its asset ID — not a fresh in-memory - build — so emissions are computed against the finalized exported - raster), loads the four GEE-side input datasets, then stitches per-epoch - LUC + peatland-conversion bands, the 2020 peatland-occupation band, and - the two 2020-allocated bands into a single 11-band float32 image. - - `geometry` is an `ee.Geometry` covering the requested region — - single-state, multi-state union, or CONUS. No per-state subdivision; - the whole region is built in one graph. - - Per-input ``.clip()`` calls are scoped to ``geometry.bounds()`` - rather than ``geometry`` itself: a polygon clip pays a per-vertex- - per-tile cost; a 4-vertex bbox clip does not. The polygon-shape - mask needed for the asset's content footprint is applied at the - export sink via ``fips_mask`` (see ``export_emissions_asset``), not - at the per-input clips. - - Pure graph building — no exports, no `.getInfo()`. - """ - region_bbox = geometry.bounds() - land_use = _load_land_use_bands(land_use_asset_id, region_bbox) - - agb = _load_agb(region_bbox) - inputs = EmissionsInputs( - agb=agb, - bgb=_load_bgb(region_bbox, agb_image=agb), - soc=_load_soc(region_bbox), - climate_zone=_load_ipcc_climate(region_bbox), - is_peatland=land_use.select('is_peatland'), - crops_2020=land_use.select('crops_2020'), - pixel_area_ha=_load_pixel_area_ha(), - ) - - luc_bands: list[EEImage] = [] - peat_bands: list[EEImage] = [] - for from_year, to_year in GLAD_EPOCH_PAIRS: - encoded = land_use.select(transitions_band(from_year, to_year)) - epoch_image = calculate_epoch_emissions(encoded, inputs, (from_year, to_year)) - luc_bands.append(epoch_image.select(luc_emissions_band(from_year, to_year))) - peat_bands.append( - epoch_image.select(peatland_conversion_band(from_year, to_year)) - ) - - occupation = calculate_peatland_occupation(inputs) - allocated = calculate_allocated_emissions_2020(luc_bands, peat_bands, occupation) - - image = luc_bands[0] - for band in luc_bands[1:] + peat_bands + [occupation, allocated]: - image = image.addBands(band) - - return image.toFloat().select(EMISSIONS_BAND_NAMES) - - -# --------------------------------------------------------------------------- -# Export -# --------------------------------------------------------------------------- - - -def export_emissions_asset( - image: EEImage, - region: str, - version: str, - geometry: EEGeometry, - fips_mask: EEImage, - force: bool = False, -) -> ee.batch.Task | None: - """Export the emissions image as a GEE asset. - - See ``transform._export.export_image_asset_to_gee`` for the canonical - export shape (``region=geometry.bounds()``, GLAD CRS pinning, - cache-probe semantics). Returns the task handle for polling, or - None on cache hit. - """ - return export_image_asset_to_gee( - image=image, - asset_kind='emissions', - region=region, - version=version, - geometry=geometry, - fips_mask=fips_mask, - force=force, - ) diff --git a/jdluc/transform/land_use.py b/jdluc/transform/land_use.py deleted file mode 100644 index 9c09678..0000000 --- a/jdluc/transform/land_use.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Per-state land use raster construction. - -Builds the 6-band land_use_{region}_{version} raster for a state: - - 4 packed transition bands (uint8, one per GLAD epoch pair) - - crops_2020 (uint8, CDL crop code on GLAD cropland pixels) - - is_peatland (uint8, binary peatland mask) - -See specs/pipeline_tech_design.md § land_use.py for details. -""" - -import logging - -import ee - -from jdluc.transform._export import export_image_asset_to_gee -from jdluc.utils._ee_types import EEGeometry, EEImage -from jdluc.utils.constants import ( - DATASET_INVENTORY, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - GLAD_EPOCH_PAIRS, - GLAD_EPOCH_YEARS, - LAND_USE_CATEGORIES, - transitions_band, -) -from jdluc.utils.transitions import encode_transition_image - -logger = logging.getLogger(__name__) - -# Output band schema for the land_use raster. -LAND_USE_BAND_NAMES: list[str] = [ - transitions_band(fy, ty) for fy, ty in GLAD_EPOCH_PAIRS -] + ['crops_2020', 'is_peatland'] - - -# --------------------------------------------------------------------------- -# Classification -# --------------------------------------------------------------------------- - - -def classify_glad_glc(raw_image: EEImage) -> EEImage: - """Reclassify a raw GLAD GLCLUC v2 band into 9 land use categories. - - Takes a single-band uint8 image with raw GLAD values (0-250) and returns - a single-band uint8 image with category codes 1-9 (0 for unclassified). - """ - raw_values: list[int] = [] - category_codes: list[int] = [] - for cat in LAND_USE_CATEGORIES.values(): - for v in cat.glad_values: - raw_values.append(v) - category_codes.append(cat.code) - - return ( - raw_image.remap(raw_values, category_codes, defaultValue=0) - .uint8() - .rename('category') - ) - - -# --------------------------------------------------------------------------- -# Auxiliary band loaders -# --------------------------------------------------------------------------- - - -def _load_crops_2020( - state_geometry: EEGeometry, glad_category_2020: EEImage -) -> EEImage: - """Load CDL 2020 crop codes, mask to GLAD cropland pixels. - - CDL is on a non-GLAD native grid (Albers Equal Area at 30 m); the - actual reprojection to the GLAD grid happens at the export node via - the Export call's pinned ``crs`` / ``crsTransform`` (the GEE - pull-through pattern). Default nearest-neighbor resampling is the - correct choice for categorical CDL crop codes and is GEE's default - when no ``.resample(...)`` is specified. - """ - cdl_2020 = ( - ee.ImageCollection(DATASET_INVENTORY['usda_cdl']['gee_asset_id']) - .filter(ee.Filter.calendarRange(2020, 2020, 'year')) - .first() - .select('cropland') - .clip(state_geometry) - ) - cropland_mask = glad_category_2020.eq(LAND_USE_CATEGORIES['cropland'].code) - return cdl_2020.where(cropland_mask.Not(), 0).uint8().rename('crops_2020') - - -def _load_is_peatland(state_geometry: EEGeometry) -> EEImage: - """Load GFW Global Peatlands as a binary mask. - - The asset is an ``ImageCollection`` of 10°×10° tile members already - on the GLAD 30 m grid; ``.mosaic()`` combines them into a single - image lazily on read but drops the per-tile projection metadata — - ``setDefaultProjection`` re-attaches the GLAD projection without - forcing a resample (the tiles are already on that grid). The actual - grid alignment of the export output is driven by the Export call's - pinned ``crs`` / ``crsTransform`` (pull-through pattern). Categorical - (binary 0/1) — default nearest-neighbor resampling is the right - choice. - """ - return ( - ee.ImageCollection(DATASET_INVENTORY['gfw_peatlands']['gee_asset_id']) - .mosaic() - .setDefaultProjection(crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM) - .clip(state_geometry) - .gt(0) # ensure binary 0/1 - .uint8() - .rename('is_peatland') - ) - - -# --------------------------------------------------------------------------- -# Main graph builder -# --------------------------------------------------------------------------- - - -def build_land_use_image(geometry: EEGeometry) -> EEImage: - """Construct the 6-band land_use raster for the requested region. - - Bands (all uint8): - transitions_2000_2005, transitions_2005_2010, - transitions_2010_2015, transitions_2015_2020, - crops_2020, is_peatland - - `geometry` is an `ee.Geometry` covering the requested region — a - single state's boundary, the union of multiple states, or CONUS. No - per-state subdivision; the whole region is built in one graph. - - Per-input ``.clip()`` calls are scoped to ``geometry.bounds()`` - rather than ``geometry`` itself: a polygon clip pays a per-vertex- - per-tile cost; a 4-vertex bbox clip does not. The polygon-shape - mask needed for the asset's content footprint is applied at the - export sink via ``fips_mask`` (see ``export_land_use_asset``), not - at the per-input clips. - - Pure graph building -- no Export calls. - """ - region_bbox = geometry.bounds() - - # Load and classify all five GLAD epochs - classified: dict[int, EEImage] = {} - for year in GLAD_EPOCH_YEARS: - asset_id = DATASET_INVENTORY['glad_glcluc_v2']['gee_asset_id'].format(year=year) - raw = ee.Image(asset_id).clip(region_bbox) - classified[year] = classify_glad_glc(raw) - - # Build the four transition bands - transition_bands: list[EEImage] = [] - for from_year, to_year in GLAD_EPOCH_PAIRS: - band = encode_transition_image( - classified[from_year], classified[to_year] - ).rename(transitions_band(from_year, to_year)) - transition_bands.append(band) - - # Stack transitions - image = transition_bands[0] - for band in transition_bands[1:]: - image = image.addBands(band) - - # Add crops_2020 band - crops = _load_crops_2020(region_bbox, classified[2020]) - image = image.addBands(crops) - - # Add is_peatland band - peatland = _load_is_peatland(region_bbox) - image = image.addBands(peatland) - - # Enforce uint8 on all bands and verify band order - image = image.uint8().select(LAND_USE_BAND_NAMES) - - return image - - -# --------------------------------------------------------------------------- -# Export -# --------------------------------------------------------------------------- - - -def export_land_use_asset( - image: EEImage, - region: str, - version: str, - geometry: EEGeometry, - fips_mask: EEImage, - force: bool = False, -) -> ee.batch.Task | None: - """Export the land_use image as a GEE asset. - - See ``transform._export.export_image_asset_to_gee`` for the canonical - export shape (``region=geometry.bounds()``, GLAD CRS pinning, - cache-probe semantics). Returns the task handle for polling, or - None on cache hit. - """ - return export_image_asset_to_gee( - image=image, - asset_kind='land_use', - region=region, - version=version, - geometry=geometry, - fips_mask=fips_mask, - force=force, - ) diff --git a/jdluc/transform/summary_tables.py b/jdluc/transform/summary_tables.py deleted file mode 100644 index 96a60c7..0000000 --- a/jdluc/transform/summary_tables.py +++ /dev/null @@ -1,1033 +0,0 @@ -"""Per-state summary-table construction. - -Reduces the cached land_use and emissions rasters to two GEE -FeatureCollection table assets per state: transitions_{region}_{version} -at (county_fips, epoch_transition, emissions_type) grain, and -crops_{region}_{version} at (county_fips, crop_code) grain. - -See specs/pipeline_tech_design.md § summary_tables.py for details. -""" - -import functools -import logging -from collections.abc import Mapping, Sequence -from typing import Any - -import ee - -from jdluc.utils._ee_types import EEFeatureCollection, EEGeometry, EEImage -from jdluc.utils.asset_management import delete_asset_safely -from jdluc.utils.constants import ( - ALL_ROW_CROP_CODES, - BUSHEL_WEIGHT_KG, - CROP_CODE_TO_GROUP, - CROP_DRIVER_MAPPING, - CROP_GROUPS, - CROPS_TABLE_COLUMNS, - EMISSIONS_TYPE_CODE_TO_NAME, - EMISSIONS_TYPE_CODES, - EMISSIONS_TYPE_NAMES, - EMISSIVE_ENCODED_CODES, - GEE_NASS_YIELDS, - GHGP_EPOCH_WEIGHTS_2020, - GLAD_CRS, - GLAD_CRS_TRANSFORM, - GLAD_EPOCH_PAIRS, - HA_PER_ACRE, - NASS_TO_CROP_GROUP, - NASS_YIELD_YEARS, - PEATLAND_DRAINAGE_ENCODED_CODES, - PEATLAND_OCCUPATION_EPOCH_LABEL, - TRANSITIONS_TABLE_COLUMNS, - luc_emissions_band, - output_asset_id, - peatland_conversion_band, - transitions_band, -) -from jdluc.utils.gee import asset_exists - -logger = logging.getLogger(__name__) - -# Pulled from the central emissions_type vocabulary so the reducer's group -# keys always match the 1..11 emissions-type codes. -_LUC_CODES: list[int] = list(range(1, 10)) -_PEATLAND_CONVERSION_CODE: int = EMISSIONS_TYPE_CODES['peatland_conversion'] -_PEATLAND_OCCUPATION_CODE: int = EMISSIONS_TYPE_CODES['peatland_occupation'] - -# --------------------------------------------------------------------------- -# Crops-table driver-pair encoded transitions -# --------------------------------------------------------------------------- -# Subsets of EMISSIVE_ENCODED_CODES partitioned by source family per -# CROP_DRIVER_MAPPING. Used in `_build_crops_metric_stack` to construct the -# per-pixel `is_forest_pair` / `is_short_veg_pair` masks via remap. Built -# once at module load from public constants — the source-family partitioning -# is fully derivable from EMISSIVE_ENCODED_CODES (ordered) + EMISSIONS_TYPE_NAMES -# (parallel list, first 9 are LUC names) + CROP_DRIVER_MAPPING. -_FOREST_ENCODED_CODES: list[int] = [ - EMISSIVE_ENCODED_CODES[i] - for i in range(len(EMISSIVE_ENCODED_CODES)) - if CROP_DRIVER_MAPPING[EMISSIONS_TYPE_NAMES[i]] == 'pct_forest' -] -_SHORT_VEG_ENCODED_CODES: list[int] = [ - EMISSIVE_ENCODED_CODES[i] - for i in range(len(EMISSIVE_ENCODED_CODES)) - if CROP_DRIVER_MAPPING[EMISSIONS_TYPE_NAMES[i]] == 'pct_short_veg' -] - -# Output column ordering for the 10-band metric stack. The reducer is -# `ee.Reducer.sum().repeat(10).group(groupField=10, ...)`; the group-key band -# (crop_code) sits at position 10. Reducer output sums come back as a -# 10-element list in this exact order — the flatten step in `_emit_crop_row` -# indexes via `sums.get(0..9)`, so any reordering breaks the pivot. -_METRIC_BAND_NAMES: tuple[str, ...] = ( - 'area_total', - 'area_peatland', - 'allocated_forest', - 'allocated_short_veg', - 'allocated_peatland_conversion', - 'allocated_peatland_occupation', - 'allocated_2005', - 'allocated_2010', - 'allocated_2015', - 'allocated_2020', -) - - -# --------------------------------------------------------------------------- -# Server-side lookup dictionaries (lazy) -# --------------------------------------------------------------------------- -# `build_transitions_table` (and the future server-side `build_crops_table`) -# attaches derived columns via `.map()` chains that do `ee.Dictionary` lookups -# server-side. Build the dictionaries once per process; lazy because `ee` may -# not be initialized at module import time. - - -@functools.lru_cache(maxsize=1) -def _emissions_type_name_dict() -> ee.Dictionary: - """`emissions_type_code (str) → emissions_type` (e.g. `'1'` → `'forest_to_cropland'`).""" - return ee.Dictionary({str(c): n for c, n in EMISSIONS_TYPE_CODE_TO_NAME.items()}) - - -@functools.lru_cache(maxsize=1) -def _ghgp_weight_dict() -> ee.Dictionary: - """`'{from}_{to}' → GHGP 20-year-lookback epoch weight` for LUC + peatland_conv rows.""" - return ee.Dictionary( - {f'{a}_{b}': w for (a, b), w in GHGP_EPOCH_WEIGHTS_2020.items()} - ) - - -@functools.lru_cache(maxsize=1) -def _crop_group_dict() -> ee.Dictionary: - """`crop_code (str) → crop_group` (e.g. `'1'` → `'corn'`, `'22'` → `'wheat'`).""" - return ee.Dictionary({str(c): g for c, g in CROP_CODE_TO_GROUP.items()}) - - -@functools.lru_cache(maxsize=1) -def _flat_yield_dict() -> dict[str, dict[str, float]]: - """Build the per-(state_fips,crop_code) yield dict once per process.""" - return _nass_yield_dict_from_features(_load_nass_yield_features(GEE_NASS_YIELDS)) - - -@functools.lru_cache(maxsize=1) -def _yield_bu_dict() -> ee.Dictionary: - """`'{state_fips}|{crop_code}' → bu/acre yield` (state-level, NASS_YIELD_YEARS mean).""" - return ee.Dictionary( - {k: v['yield_bu_per_acre'] for k, v in _flat_yield_dict().items()} - ) - - -@functools.lru_cache(maxsize=1) -def _yield_kg_dict() -> ee.Dictionary: - """`'{state_fips}|{crop_code}' → kg/ha yield` (state-level, NASS_YIELD_YEARS mean).""" - return ee.Dictionary( - {k: v['yield_kg_per_ha'] for k, v in _flat_yield_dict().items()} - ) - - -def _dict_get_or_null(dictionary: Any, key: Any) -> Any: - """`ee.Dictionary.get(key, None)` doesn't do what you'd hope. - - GEE's `Dictionary.get(key, defaultValue)` treats a `None`/`null` - `defaultValue` as "no default — raise on missing key" rather than as - "return null on missing key." To actually get null-on-miss semantics - (which the crops table needs for missing NASS yields and the - transitions decoration needs for the crop_code=0 sentinel), branch - explicitly via `ee.Algorithms.If(dict.contains(key), dict.get(key), None)`. - """ - d = ee.Dictionary(dictionary) - return ee.Algorithms.If(d.contains(key), d.get(key), None) - - -# --------------------------------------------------------------------------- -# NASS yield dictionary -# --------------------------------------------------------------------------- -# The NASS asset (extract/nass_yields.py) carries one feature per raw -# (state_fips, commodity_desc, unit_desc, agg_level_desc, year) row from the -# USDA QuickStats archive, pre-filtered to the three commodity families -# (CORN / SOYBEANS / WHEAT) and the YIELD statistic. This module does the -# rest of the pipeline: (a) filter to state-level bu/acre rows within -# NASS_YIELD_YEARS, (b) take the arithmetic mean per (state, commodity), -# (c) convert bu/acre -> kg/ha via BUSHEL_WEIGHT_KG / HA_PER_ACRE, and -# (d) fan out the single WHEAT row to the three wheat CDL codes so -# downstream lookups can key on crop_code directly. Keeping these steps -# at the transform layer makes the window / conversion choices visible -# in one place and keeps the extract asset window-agnostic (a new -# NASS_YIELD_YEARS window does not force a re-extract). - - -@functools.lru_cache(maxsize=1) -def _load_nass_yield_features(asset_id: str) -> tuple[dict[str, Any], ...]: - """Materialize the NASS yield FeatureCollection to a tuple of property dicts. - - The NASS extract asset carries the full QuickStats archive - pre-filtered to YIELD + (CORN/SOYBEANS/WHEAT) — order of 10⁶ rows. - Pulling the whole thing through ``.getInfo()`` blows past GEE's - response-size cap, so we apply the methodology's - ``agg_level_desc == 'STATE'`` / ``unit_desc == 'BU / ACRE'`` / - ``year ∈ NASS_YIELD_YEARS`` / ``commodity_desc ∈ NASS_TO_CROP_GROUP`` - filters server-side first. After filtering, the result is on the - order of 50 states × 4 years × 3 commodities ≈ 600 features (a - few hundred KB of JSON), which round-trips cheaply. - - The year window is baked in via ``NASS_YIELD_YEARS``: the asset - version tag (``EXPECTED_NASS_VERSION = 'v2017_2020'``) already - encodes that coupling, so a methodology window change forces a - new asset ID, which invalidates this cache. - - Cached at module scope (keyed on the asset ID) so a multi-state - run does the load once and reuses across every state's - ``build_crops_table()`` call. - """ - fc = ( - ee.FeatureCollection(asset_id) - .filter(ee.Filter.eq('agg_level_desc', 'STATE')) - .filter(ee.Filter.eq('unit_desc', 'BU / ACRE')) - .filter(ee.Filter.inList('commodity_desc', list(NASS_TO_CROP_GROUP))) - .filter(ee.Filter.inList('year', list(NASS_YIELD_YEARS))) - ) - raw = fc.getInfo() - features = tuple(feature['properties'] for feature in raw['features']) - logger.info( - f'NASS yield load: {asset_id} → {len(features)} feature(s) ' - f'after server-side filter' - ) - return features - - -def _filter_average_convert_nass_rows( - rows: Sequence[Mapping[str, Any]], - *, - years: Sequence[int] = NASS_YIELD_YEARS, -) -> dict[tuple[str, str], dict[str, float]]: - """Turn raw NASS rows into a `(state_fips, commodity) → yield dict`. - - Applies the methodology's state-level yield-mean definition: - - - Keep rows with ``agg_level_desc == 'STATE'`` and - ``unit_desc == 'BU / ACRE'`` and ``year in years``. - - Group by ``(state_fips, commodity_desc)`` and take the arithmetic - mean of ``value_bu_per_acre`` (4-year mean by default per - methodology § Calculate crop yields). - - Convert to kg/ha via ``yield_bu_per_acre * BUSHEL_WEIGHT_KG / HA_PER_ACRE``. - - Unknown ``commodity_desc`` values (not in ``NASS_TO_CROP_GROUP``) are - dropped silently — the extract stage pre-filters to the three - methodology commodities, so anything else is genuinely noise. - """ - year_set = {int(y) for y in years} - accum: dict[tuple[str, str], list[float]] = {} - for props in rows: - if str(props.get('agg_level_desc')) != 'STATE': - continue - if str(props.get('unit_desc')) != 'BU / ACRE': - continue - try: - year = int(props['year']) - except (KeyError, TypeError, ValueError): - continue - if year not in year_set: - continue - commodity = str(props.get('commodity_desc', '')) - if commodity not in NASS_TO_CROP_GROUP: - continue - try: - value = float(props['value_bu_per_acre']) - except (KeyError, TypeError, ValueError): - continue - state_fips = str(props['state_fips']).zfill(2) - accum.setdefault((state_fips, commodity), []).append(value) - - result: dict[tuple[str, str], dict[str, float]] = {} - for key, values in accum.items(): - if not values: - continue - mean_bu_per_acre = sum(values) / len(values) - commodity = key[1] - kg_per_ha = mean_bu_per_acre * BUSHEL_WEIGHT_KG[commodity] / HA_PER_ACRE - result[key] = { - 'yield_bu_per_acre': mean_bu_per_acre, - 'yield_kg_per_ha': kg_per_ha, - } - return result - - -def _nass_yield_dict_from_features( - features: Sequence[Mapping[str, Any]], - *, - years: Sequence[int] = NASS_YIELD_YEARS, -) -> dict[str, dict[str, float]]: - """Build the `{f"{state_fips}|{crop_code}": {yield_*}}` dict. - - Delegates the filter / average / convert to - ``_filter_average_convert_nass_rows``, then fans the single WHEAT - row per state out to CDL codes 22 / 23 / 24 so downstream lookups - can key on crop_code directly. - - Separated from ``_build_nass_yield_dictionary`` so unit tests can - exercise the fan-out and key-formatting on synthetic rows without - initializing the GEE client. - """ - averaged = _filter_average_convert_nass_rows(features, years=years) - mapping: dict[str, dict[str, float]] = {} - for (state_fips, commodity), yield_values in averaged.items(): - crop_name = NASS_TO_CROP_GROUP[commodity] - for code in CROP_GROUPS[crop_name]: - mapping[f'{state_fips}|{code}'] = yield_values - return mapping - - -# --------------------------------------------------------------------------- -# Transitions table -# --------------------------------------------------------------------------- -# One per-county row per (epoch_transition, emissions_type) grain, per -# specs/pipeline_tech_design.md § Tables > transitions. Built from ten -# grouped reduceRegions calls (two per GLAD epoch pair for LUC + -# peatland_conversion, plus one for peatland_occupation). Each reducer -# emits one feature per county whose `groups` property carries -# (group_key, [area_ha, emissions_tco2]) tuples; a server-side `.map()` -# flattens those into per-(county, type) features tagged with the -# constant `epoch_transition` for that reducer. The ten flattened FCs -# `.merge()` server-side, a final `.map()` attaches `emissions_type` and -# `allocated_emissions_2020_tco2` (via the lazy `ee.Dictionary` lookups -# above), zero rows are dropped, and the result is exported directly via -# `Export.table.toAsset` — no `.getInfo()` round-trips during graph build. - - -def _pixel_area_ha_image() -> EEImage: - """Per-pixel area in hectares, pinned to the GLAD grid. - - Matches the helper in transform/emissions.py so pixel areas used in the - emissions raster and in these summary tables agree to the digit. - """ - return ( - ee.Image.pixelArea() - .divide(1e4) - .reproject(crs=GLAD_CRS, crsTransform=GLAD_CRS_TRANSFORM) - .rename('area_ha') - ) - - -# Composite-key encoding for the transitions reducer: -# fips × 100 + emissions_type_code packs (county, type) into a single int32. -# fips ∈ [1001, 56045]; emissions_type_code ∈ [1, 11]; composite ∈ -# [100101, 5604511] — well within int32. Decoded in -# `_flatten_composite_groups_two_band` via integer divide / mod. -_TRANSITIONS_COMPOSITE_MULT: int = 100 - - -def _reduce_grouped_two_band_with_fips( - pixel_area_ha: EEImage, - emissions: EEImage, - group_key: EEImage, - fips_band: EEImage, - region_bbox: EEGeometry, -) -> Any: - """Stack `(area_ha, emissions, composite_key)` and grouped-reduce server-side. - - Replaces the pre-Phase-11 ``reduceRegions(collection=counties)`` shape - with a single ``reduceRegion(geometry=region_bbox)`` whose group field - is a composite key encoding both county-FIPS and emissions-type-code. - Investigation doc § "Polygon-shape overhead was the driver" measured - the polygon-vertex-per-tile cost the reduceRegions(counties) pattern - paid as ~89% of pre-Phase-11 transform cost; this rewrite eliminates - it by reading the per-county dimension off a raster band rather than - iterating polygon features. - - Returns the raw server-side ``ee.List`` of group dicts (each - ``{composite_key, sum: [area_ha, emissions_tco2]}``). The caller - flattens with ``_flatten_composite_groups_two_band``. - - `tileScale=4` keeps per-tile memory hedged. - """ - mask = group_key.gt(0).And(fips_band.gt(0)) - composite_key = ( - fips_band.multiply(_TRANSITIONS_COMPOSITE_MULT).add(group_key).updateMask(mask) - ) - image = ( - pixel_area_ha.updateMask(mask) - .rename('area_ha') - .addBands(emissions.updateMask(mask).rename('emis')) - .addBands(composite_key.rename('composite_key')) - ) - reducer = ee.Reducer.sum().repeat(2).group(groupField=2, groupName='composite_key') - result = image.reduceRegion( - reducer=reducer, - geometry=region_bbox, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e13), - tileScale=4, - ) - return ee.List(result.get('groups')) - - -def _flatten_composite_groups_two_band( - groups: Any, - epoch_label: str, -) -> EEFeatureCollection: - """Flatten composite-key group dicts into per-(county, type) features. - - `groups` is the server-side ee.List returned by - `_reduce_grouped_two_band_with_fips`. Each entry is - `{composite_key, sum: [area_ha, emissions_tco2]}` where - ``composite_key = fips × 100 + emissions_type_code``. Output features - decode the composite back to the canonical (5-digit zero-padded - `county_fips` string, `emissions_type_code` int) pair the - pre-Phase-11 schema produced. - - Pure server-side: no `.getInfo()`. Output features carry - placeholder Point(0, 0) geometry — `Export.table.toAsset` rejects - null-geometry features, and the summary tables are tabular so any - geometry would do. - """ - epoch_label_ee = ee.String(epoch_label) - - def _emit(g: Any) -> Any: - d = ee.Dictionary(g) - composite = ee.Number(d.get('composite_key')) - fips = composite.divide(_TRANSITIONS_COMPOSITE_MULT).floor().toInt() - emissions_type_code = composite.mod(_TRANSITIONS_COMPOSITE_MULT).toInt() - sums = ee.List(d.get('sum')) - return ee.Feature( - ee.Geometry.Point(0, 0), - { - # Match the pre-Phase-11 5-digit zero-padded string schema - # so downstream BigQuery consumers see no - # type drift in the county_fips column. - 'county_fips': fips.format('%05d'), - 'epoch_transition': epoch_label_ee, - 'emissions_type_code': emissions_type_code, - 'total_area_ha': ee.Number(sums.get(0)), - 'total_emissions_tco2': ee.Number(sums.get(1)), - }, - ) - - return ee.FeatureCollection(groups.map(_emit)) - - -def _decorate_transitions_row(feature: Any) -> Any: - """Attach `emissions_type` (name) and `allocated_emissions_2020_tco2`. - - Server-side counterpart of the pre-Phase-8 client-side pandas - decoration. `peatland_occupation` rows get full-weight allocation - (annual emission, no GHGP lookback discount); LUC + peatland_conversion - rows get the GHGP per-epoch weight via the `_ghgp_weight_dict` lookup. - """ - code = ee.Number(feature.get('emissions_type_code')) - epoch = ee.String(feature.get('epoch_transition')) - total = ee.Number(feature.get('total_emissions_tco2')) - weight = ee.Algorithms.If( - code.eq(_PEATLAND_OCCUPATION_CODE), - ee.Number(1.0), - ee.Number(_ghgp_weight_dict().get(epoch, 0)), - ) - return feature.set( - { - # `_dict_get_or_null` covers code=0 sentinel rows emitted by the - # group_key=0 multiply-mask zero-out — they're filtered downstream - # but the .map() must not raise on them first. - 'emissions_type': _dict_get_or_null( - _emissions_type_name_dict(), code.format('%d') - ), - 'allocated_emissions_2020_tco2': total.multiply(ee.Number(weight)), - } - ) - - -def build_transitions_table( - land_use_asset: EEImage, - emissions_asset: EEImage, - fips_band: EEImage, - region_bbox: EEGeometry, -) -> EEFeatureCollection: - """Build the `transitions` summary FeatureCollection server-side. - - Ten composite-key grouped reducers (two per GLAD epoch pair for LUC + - peatland_conversion, plus one for peatland_occupation) each emit a - server-side ``ee.List`` of group dicts keyed on - ``fips × 100 + emissions_type_code``. Each list is flattened to - per-(county, type) features tagged with the constant `epoch_transition` - for that reducer; `.merge()` chains the ten flattened FCs; - `.map(_decorate_transitions_row)` attaches the `emissions_type` name - and `allocated_emissions_2020_tco2` weight; zero rows are filtered; - `.select(...)` projects to the canonical column set. No `.getInfo()` - round-trips during graph build. - - ``fips_band`` is the run-scoped FIPS-valued image (county FIPS where - the pixel is in one of the run's CONUS counties, masked elsewhere) — - it carries both the per-county grouping dimension and the implicit - polygon-shape mask. ``region_bbox`` is the bounding rectangle of the - run's region; the reducer iterates pixels inside it. The bbox+mask - pattern avoids the polygon-vertex-per-tile cost of - ``reduceRegions(collection=counties)``. - """ - pixel_area = _pixel_area_ha_image() - is_peatland = land_use_asset.select('is_peatland') - crops_2020 = land_use_asset.select('crops_2020') - - flattened: list[EEFeatureCollection] = [] - - # LUC + peatland_conversion: two reducers per epoch. - for from_y, to_y in GLAD_EPOCH_PAIRS: - epoch_label = f'{from_y}_{to_y}' - encoded = land_use_asset.select(transitions_band(from_y, to_y)) - - luc_group_key = encoded.remap( - EMISSIVE_ENCODED_CODES, _LUC_CODES, defaultValue=0 - ) - flattened.append( - _flatten_composite_groups_two_band( - _reduce_grouped_two_band_with_fips( - pixel_area_ha=pixel_area, - emissions=emissions_asset.select(luc_emissions_band(from_y, to_y)), - group_key=luc_group_key, - fips_band=fips_band, - region_bbox=region_bbox, - ), - epoch_label=epoch_label, - ) - ) - - pc_group_key = encoded.remap( - PEATLAND_DRAINAGE_ENCODED_CODES, - [_PEATLAND_CONVERSION_CODE] * len(PEATLAND_DRAINAGE_ENCODED_CODES), - defaultValue=0, - ).multiply(is_peatland) - flattened.append( - _flatten_composite_groups_two_band( - _reduce_grouped_two_band_with_fips( - pixel_area_ha=pixel_area, - emissions=emissions_asset.select( - peatland_conversion_band(from_y, to_y) - ), - group_key=pc_group_key, - fips_band=fips_band, - region_bbox=region_bbox, - ), - epoch_label=epoch_label, - ) - ) - - # Peatland_occupation: single reducer, constant code on peatland × cropland. - po_group_key = is_peatland.multiply(crops_2020.gt(0)).multiply( - _PEATLAND_OCCUPATION_CODE - ) - flattened.append( - _flatten_composite_groups_two_band( - _reduce_grouped_two_band_with_fips( - pixel_area_ha=pixel_area, - emissions=emissions_asset.select('peatland_occupation_2020'), - group_key=po_group_key, - fips_band=fips_band, - region_bbox=region_bbox, - ), - epoch_label=PEATLAND_OCCUPATION_EPOCH_LABEL, - ) - ) - - # Merge all 10 flattened FCs server-side. - merged = flattened[0] - for fc in flattened[1:]: - merged = merged.merge(fc) - - # Decorate, then drop both zero-contribution rows and group_key=0 - # sentinel rows (the latter are emitted by the .multiply(mask) zero-out - # in _reduce_grouped_two_band; they always carry area=emis=0 anyway, - # but the explicit code>0 filter keeps the schema unambiguous). - decorated = merged.map(_decorate_transitions_row) - nonzero = decorated.filter( - ee.Filter.Or( - ee.Filter.gt('total_area_ha', 0), - ee.Filter.gt('total_emissions_tco2', 0), - ) - ).filter(ee.Filter.gt('emissions_type_code', 0)) - - # Project to canonical column set; emissions_type_code is internal. - return nonzero.select(propertySelectors=list(TRANSITIONS_TABLE_COLUMNS)) - - -# --------------------------------------------------------------------------- -# Crops table -# --------------------------------------------------------------------------- -# One per-county row per crop_code grain. Built from a single composite- -# key grouped `reduceRegion` over a 10-band metric stack + a composite- -# key band — each metric band is a per-pixel contribution to one output -# column (area_total, area_peatland, allocated_, -# allocated_) by construction, so the reducer sums plus a server- -# side flatten produce per-(county, crop) totals + the four -# `pct_` and four `pct_epoch_*` shares without any cross-row -# pivot. NASS yield join via `ee.Dictionary.get(key, None)`; null-safe -# `pct_*` and `emissions_factor_kgco2e_per_kg` via `ee.Algorithms.If` -# fallbacks (`divide(0)` would otherwise serialize as integer 0, not null). - -# Composite-key encoding for the crops reducer: -# fips × 1000 + crop_code packs (county, crop) into a single int32. CDL -# crop codes go up to 254, so 3 digits are required (vs the transitions -# table's 2). Max composite = 56045 × 1000 + 254 = 56,045,254 — well -# within int32. Decoded in `_emit_crop_row` via integer divide / mod. -_CROPS_COMPOSITE_MULT: int = 1000 - - -def _build_crops_metric_stack( - land_use_asset: EEImage, - emissions_asset: EEImage, - fips_band: EEImage, -) -> EEImage: - """Build the 11-band image consumed by the single crops-table reducer. - - Bands 0..9 are the per-pixel contributions to the 10 output metrics - (canonical order in `_METRIC_BAND_NAMES`); band 10 is the composite - key (fips × 1000 + crop_code), the grouped reducer's groupField. - Every metric band is `0` on non-row-crop pixels by construction (the - `is_row_crop` factor), and all sums are computed server-side at - `Σ_epoch (epoch_weight × source_band × driver_mask) × is_row_crop` - so the per-`(county, crop)` totals fall out of one grouped reducer - pass. - - `forest_to_short_veg` (the only emissive LUC pair whose destination - isn't cropland or built_up) is automatically excluded — `is_row_crop` - is 0 on its destination pixels because `crops_2020` is masked to GLAD - cropland in `_load_crops_2020`. No explicit guard needed. - - ``fips_band`` is the run-scoped FIPS-valued image used to compute - the composite key. Pixels outside any of the run's counties have - ``fips_band`` masked, so the composite key is masked too and the - reducer skips them. - """ - # `.unmask(0)` everywhere a band derived from the cached land_use or - # emissions asset gets multiplied into the metric stack. Without it the - # grouped reducer's per-band sampling drifts when two of the multiplied - # bands have inherited inconsistent masks (e.g. is_peatland inherits the - # GFW Peatlands source's mask, transitions / crops_2020 don't), which silently - # under-counts on some county polygons. - pixel_area = _pixel_area_ha_image() - is_peatland = land_use_asset.select('is_peatland').unmask(0) - crops_2020 = land_use_asset.select('crops_2020').unmask(0) - is_row_crop = crops_2020.remap( - ALL_ROW_CROP_CODES, [1] * len(ALL_ROW_CROP_CODES), defaultValue=0 - ).unmask(0) - - # Per-pixel crop_code on row-crop pixels (0 elsewhere). Combined - # with fips_band into a composite key by the caller; not exported as - # its own band of the metric stack. - crop_code = crops_2020.multiply(is_row_crop) - - # Area metrics. - area_total = pixel_area.multiply(is_row_crop).rename('area_total') - area_peatland = ( - pixel_area.multiply(is_row_crop).multiply(is_peatland).rename('area_peatland') - ) - - # Per-epoch contribution lists (avoids the .add() accumulator pattern; - # empirically produces consistent per-(county, crop) sums across - # mathematically-equivalent driver vs. epoch decompositions, where the - # accumulator pattern silently dropped contributions on some county - # polygons). - forest_contribs: list[Any] = [] - short_veg_contribs: list[Any] = [] - pc_contribs: list[Any] = [] - epoch_contribs: dict[int, list[Any]] = {2005: [], 2010: [], 2015: [], 2020: []} - - # All per-pixel contributions are renamed to `'v'` so `ImageCollection.sum()` - # sees a homogeneous collection. - def _v(image: Any) -> Any: - return image.rename('v') - - for from_y, to_y in GLAD_EPOCH_PAIRS: - weight = GHGP_EPOCH_WEIGHTS_2020[(from_y, to_y)] - encoded = land_use_asset.select(transitions_band(from_y, to_y)).unmask(0) - luc_emis = emissions_asset.select(luc_emissions_band(from_y, to_y)).unmask(0) - pc_emis = emissions_asset.select(peatland_conversion_band(from_y, to_y)).unmask( - 0 - ) - - is_forest_pair = ( - encoded.remap( - _FOREST_ENCODED_CODES, [1] * len(_FOREST_ENCODED_CODES), defaultValue=0 - ) - ).unmask(0) - is_short_veg_pair = ( - encoded.remap( - _SHORT_VEG_ENCODED_CODES, - [1] * len(_SHORT_VEG_ENCODED_CODES), - defaultValue=0, - ) - ).unmask(0) - - # Per-driver contributions, gated to row-crop. peatland_conversion - # bands are pre-masked to is_peatland in transform/emissions.py; no - # need to re-apply that mask here. - forest_contribs.append( - _v(luc_emis.multiply(weight).multiply(is_forest_pair).multiply(is_row_crop)) - ) - short_veg_contribs.append( - _v( - luc_emis.multiply(weight) - .multiply(is_short_veg_pair) - .multiply(is_row_crop) - ) - ) - pc_contribs.append(_v(pc_emis.multiply(weight).multiply(is_row_crop))) - - # Per-epoch bucket: append LUC and PC contributions separately so the - # grouped reducer sees the same single-source-band multiplication chain - # it sees for the per-driver bands. Combining them via `.add()` and a - # single mask multiply silently undercounts on the grouped-reducer - # path (verified empirically: direct `reduceRegion` on the combined - # image gives the correct sum, but the grouped reducer misses ~30% of - # contributions on some county polygons). `ImageCollection.sum()` - # collapses both contributions per pixel before grouping. - epoch_contribs[to_y].append(_v(luc_emis.multiply(weight).multiply(is_row_crop))) - epoch_contribs[to_y].append(_v(pc_emis.multiply(weight).multiply(is_row_crop))) - - # peatland_occupation: full weight (1.0). Already gated on is_peatland - # in the source band; narrow further to is_row_crop here. Also goes - # into the 2020 epoch bucket per metric definition. - occupation = emissions_asset.select('peatland_occupation_2020').unmask(0) - allocated_po = occupation.multiply(is_row_crop) - epoch_contribs[2020].append(_v(allocated_po)) - - allocated_forest = ee.ImageCollection(forest_contribs).sum() - allocated_short_veg = ee.ImageCollection(short_veg_contribs).sum() - allocated_pc = ee.ImageCollection(pc_contribs).sum() - allocated_2005 = ee.ImageCollection(epoch_contribs[2005]).sum() - allocated_2010 = ee.ImageCollection(epoch_contribs[2010]).sum() - allocated_2015 = ee.ImageCollection(epoch_contribs[2015]).sum() - allocated_2020 = ee.ImageCollection(epoch_contribs[2020]).sum() - - # Composite key: (fips × 1000 + crop_code), masked outside the run's - # counties (fips_band's mask) AND off non-row-crop pixels. The reducer - # skips masked pixels, so non-row-crop / out-of-region pixels never - # appear in any output group. fips_band is unmasked (`.unmask(0)`) for - # the multiply so we don't drop pixels where fips_band's mask differs - # from crops_2020's mask; the final mask is the explicit `valid` band. - valid = fips_band.gt(0).And(is_row_crop.eq(1)) - composite_key = ( - fips_band.unmask(0) - .multiply(_CROPS_COMPOSITE_MULT) - .add(crop_code) - .updateMask(valid) - .rename('composite_key') - ) - - stack = ( - area_total.addBands(area_peatland) - .addBands(allocated_forest.rename('allocated_forest')) - .addBands(allocated_short_veg.rename('allocated_short_veg')) - .addBands(allocated_pc.rename('allocated_peatland_conversion')) - .addBands(allocated_po.rename('allocated_peatland_occupation')) - .addBands(allocated_2005.rename('allocated_2005')) - .addBands(allocated_2010.rename('allocated_2010')) - .addBands(allocated_2015.rename('allocated_2015')) - .addBands(allocated_2020.rename('allocated_2020')) - .addBands(composite_key) - ) - # `.unmask(0)` on the metric bands neutralizes per-band mask drift - # (peatland-derived bands inherit the GFW Peatlands source's mask, - # others don't); the composite_key band keeps its mask so the - # reducer skips out-of-region / non-row-crop pixels. - return stack.unmask(0).updateMask(stack.select('composite_key').mask()) - - -def _emit_crop_row(group_dict: Any) -> Any: - """Server-side `ee.Feature` emission for one (county, crop) row. - - Reads the 10 reducer-sum values out of the group's `sum` list (canonical - order per `_METRIC_BAND_NAMES`), decodes the composite key into - (county_fips, crop_code), looks up the crop_group + state-level - NASS yields via `ee.Dictionary`, and computes total_production_*, - emissions_factor, and the eight `pct_*` columns. All divisions go - through the `If(denom > 0, ratio, None)` fallback because GEE coerces - `divide(0)` to integer 0 in the asset rather than null (see DD6). - - The group key is the ``fips × 1000 + crop_code`` composite produced - by `_build_crops_metric_stack`; this function decodes it via integer - divide / mod. ``county_fips`` is emitted as a 5-digit zero-padded - string; ``state_fips`` is derived from its first two digits. - """ - g = ee.Dictionary(group_dict) - composite = ee.Number(g.get('composite_key')) - fips_int = composite.divide(_CROPS_COMPOSITE_MULT).floor().toInt() - county_fips = fips_int.format('%05d') - state_fips = ee.String(county_fips).slice(0, 2) - crop_code = composite.mod(_CROPS_COMPOSITE_MULT).toInt() - sums = ee.List(g.get('sum')) - - area_total = ee.Number(sums.get(0)) - area_peatland = ee.Number(sums.get(1)) - alloc_forest = ee.Number(sums.get(2)) - alloc_short_veg = ee.Number(sums.get(3)) - alloc_pc = ee.Number(sums.get(4)) - alloc_po = ee.Number(sums.get(5)) - alloc_2005 = ee.Number(sums.get(6)) - alloc_2010 = ee.Number(sums.get(7)) - alloc_2015 = ee.Number(sums.get(8)) - alloc_2020 = ee.Number(sums.get(9)) - - total_allocated = alloc_forest.add(alloc_short_veg).add(alloc_pc).add(alloc_po) - - crop_code_str = crop_code.format('%d') - yield_key = ee.String(state_fips).cat('|').cat(crop_code_str) - yield_bu = _dict_get_or_null(_yield_bu_dict(), yield_key) - yield_kg = _dict_get_or_null(_yield_kg_dict(), yield_key) - - # Production: null when state-level NASS yield is missing. (yield_bu / - # yield_kg are server-side numeric-or-null; If treats null as falsy and - # picks the else branch.) - production_kg = ee.Algorithms.If( - yield_kg, area_total.multiply(ee.Number(yield_kg)), None - ) - production_bu = ee.Algorithms.If( - yield_bu, - area_total.multiply(ee.Number(yield_bu)).divide(HA_PER_ACRE), - None, - ) - - # EF: null when production is null OR zero. Area > 0 is filtered at the - # FC level, so production = 0 only happens when yield is null - # (handled by the outer If). - ef = ee.Algorithms.If( - production_kg, - total_allocated.multiply(1000).divide(ee.Number(production_kg)), - None, - ) - - def _safe_share(numer: Any) -> Any: - return ee.Algorithms.If( - total_allocated.gt(0), numer.divide(total_allocated), None - ) - - return ee.Feature( - ee.Geometry.Point(0, 0), - { - 'county_fips': county_fips, - 'crop_code': crop_code, - # Default `None` covers crop_code=0 (non-row-crop sentinel from the - # group_key=0 reducer output) — filtered downstream by - # `Filter.gt('total_crop_area_ha', 0)` and `Filter.neq('crop_group', None)`. - 'crop_group': _dict_get_or_null(_crop_group_dict(), crop_code_str), - 'total_crop_area_ha': area_total, - 'peatland_crop_area_ha': area_peatland, - 'yield_bu_per_acre': yield_bu, - 'yield_kg_per_ha': yield_kg, - 'total_production_kg': production_kg, - 'total_production_bu': production_bu, - 'total_allocated_emissions_tco2': total_allocated, - 'emissions_factor_kgco2e_per_kg': ef, - 'pct_forest': _safe_share(alloc_forest), - 'pct_short_veg': _safe_share(alloc_short_veg), - 'pct_peatland_conversion': _safe_share(alloc_pc), - 'pct_peatland_occupation': _safe_share(alloc_po), - 'pct_epoch_2005': _safe_share(alloc_2005), - 'pct_epoch_2010': _safe_share(alloc_2010), - 'pct_epoch_2015': _safe_share(alloc_2015), - 'pct_epoch_2020': _safe_share(alloc_2020), - }, - ) - - -def build_crops_table( - land_use_asset: EEImage, - emissions_asset: EEImage, - fips_band: EEImage, - region_bbox: EEGeometry, -) -> EEFeatureCollection: - """Build the `crops` summary FeatureCollection server-side. - - Single composite-key grouped ``reduceRegion`` over a 10-band metric - stack + a composite-key band built by `_build_crops_metric_stack`. - The composite key is ``fips × 1000 + crop_code``; the reducer skips - pixels where the key is masked (out-of-region or non-row-crop). - Server-side flatten via `_emit_crop_row` decodes each composite back - to (county_fips string, crop_code int) and emits per-(county, crop) - features with every derived column attached in one ``.map()`` pass. - Final filters drop zero-area rows. No ``.getInfo()`` round-trips - during graph construction. - - The bbox+mask shape (single ``reduceRegion`` with the composite key - as groupField) avoids the polygon-vertex-per-tile cost of - ``reduceRegions(collection=counties)``. - """ - stack = _build_crops_metric_stack( - land_use_asset=land_use_asset, - emissions_asset=emissions_asset, - fips_band=fips_band, - ) - reducer = ( - ee.Reducer.sum().repeat(10).group(groupField=10, groupName='composite_key') - ) - reduced = stack.reduceRegion( - reducer=reducer, - geometry=region_bbox, - crs=GLAD_CRS, - crsTransform=GLAD_CRS_TRANSFORM, - maxPixels=int(1e13), - tileScale=4, - ) - groups = ee.List(reduced.get('groups')) - - fc = ee.FeatureCollection(groups.map(_emit_crop_row)) - - return ( - fc.filter(ee.Filter.gt('total_crop_area_ha', 0)) - .filter(ee.Filter.neq('crop_group', None)) - .select(propertySelectors=list(CROPS_TABLE_COLUMNS)) - ) - - -# --------------------------------------------------------------------------- -# Region-level orchestration (compute_region_tables + helpers) -# --------------------------------------------------------------------------- -# Entry point for run_transform's third stage. Takes the already-resolved -# `counties` FeatureCollection (caller-side: `transform.py` resolves it -# once for the whole region), runs both table builders against the cached -# land_use and emissions assets, kicks off both exports in parallel so -# run_transform waits on them with max(t_transitions, t_crops) latency -# rather than sum, and returns the asset IDs plus task handles. - - -def export_table_asset( - fc: EEFeatureCollection, - asset_id: str, - force: bool = False, -) -> ee.batch.Task | None: - """Export a FeatureCollection as a GEE table asset. - - Mirrors `transform.land_use.export_land_use_asset` / - `transform.emissions.export_emissions_asset` for tables: cache probe - via `ee.data.getAsset`, optional force-delete, then - `Export.table.toAsset`. No CRS/transform — those are raster-only. - Returns the task handle for polling, or `None` if the asset already - exists and `force` is False. - """ - if not force: - try: - ee.data.getAsset(asset_id) - logger.info(f'table asset cached, skipping export: {asset_id}') - return None - except ee.EEException as e: - if 'does not exist' not in str(e): - raise - else: - delete_asset_safely(asset_id) - - task = ee.batch.Export.table.toAsset( - collection=fc, - assetId=asset_id, - description=asset_id.rsplit('/', 1)[-1], - ) - task.start() - logger.info(f'Started table export: {asset_id}') - return task - - -def compute_region_tables( - land_use_asset_id: str, - emissions_asset_id: str, - region: str, - version: str, - fips_band: EEImage, - region_bbox: EEGeometry, - force: bool = False, -) -> dict[str, Any]: - """Build and export both summary tables for the requested region. - - Returns a dict with `transitions_asset_id`, `crops_asset_id`, - `transitions_task`, and `crops_task`. Either task is `None` on cache - hit. Both exports are kicked off before either is waited on, so - `run_transform` can wait in parallel: total wall-clock = - max(transitions_task, crops_task), not sum. - - Cache-checks both target asset IDs **before** constructing either - FeatureCollection graph. The build_*_table calls trigger eager - server-side `value:compute` requests (NASS yield join, county - reductions) that take 10s of minutes per region even when the - output asset is already cached. Probing existence first turns a - fully-cached region into a near-instant skip. - - Both table builders consume ``fips_band`` as the per-county - grouping key (composite-keyed in a single ``reduceRegion`` over - ``region_bbox``) rather than iterating a vector FeatureCollection - per tile. - """ - transitions_asset_id = output_asset_id('transitions', region, version) - crops_asset_id = output_asset_id('crops', region, version) - - transitions_cached = (not force) and asset_exists(transitions_asset_id) - crops_cached = (not force) and asset_exists(crops_asset_id) - - if transitions_cached and crops_cached: - logger.info( - f'region {region!r}: transitions + crops both cached, ' - f'skipping graph construction' - ) - return { - 'transitions_asset_id': transitions_asset_id, - 'crops_asset_id': crops_asset_id, - 'transitions_task': None, - 'crops_task': None, - } - - land_use_asset = ee.Image(land_use_asset_id) - emissions_asset = ee.Image(emissions_asset_id) - - if transitions_cached: - logger.info(f'table asset cached, skipping export: {transitions_asset_id}') - transitions_task = None - else: - logger.info(f'Building transitions table for region {region!r}') - transitions_fc = build_transitions_table( - land_use_asset=land_use_asset, - emissions_asset=emissions_asset, - fips_band=fips_band, - region_bbox=region_bbox, - ) - transitions_task = export_table_asset( - fc=transitions_fc, - asset_id=transitions_asset_id, - force=force, - ) - - if crops_cached: - logger.info(f'table asset cached, skipping export: {crops_asset_id}') - crops_task = None - else: - logger.info(f'Building crops table for region {region!r}') - crops_fc = build_crops_table( - land_use_asset=land_use_asset, - emissions_asset=emissions_asset, - fips_band=fips_band, - region_bbox=region_bbox, - ) - crops_task = export_table_asset( - fc=crops_fc, - asset_id=crops_asset_id, - force=force, - ) - - return { - 'transitions_asset_id': transitions_asset_id, - 'crops_asset_id': crops_asset_id, - 'transitions_task': transitions_task, - 'crops_task': crops_task, - } diff --git a/jdluc/transform/transform.py b/jdluc/transform/transform.py deleted file mode 100644 index d697209..0000000 --- a/jdluc/transform/transform.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Transform stage entry point. - -Orchestrates four GEE export tasks per pipeline run, all scoped to the -region's union geometry: one `land_use` raster, one `emissions` raster, then -`transitions` + `crops` tables in parallel. No per-state subdivision; no -consolidation step. The unified raster export *is* the cross-state mosaic; -the unified `reduceRegions` *is* the cross-state table merge. - -See specs/pipeline_tech_design.md § Transform for details. -""" - -import dataclasses -import enum -import logging -from typing import Any - -import ee - -from jdluc.transform.emissions import ( - build_emissions_image, - export_emissions_asset, -) -from jdluc.transform.land_use import ( - build_land_use_image, - export_land_use_asset, -) -from jdluc.transform.summary_tables import compute_region_tables -from jdluc.utils.constants import ( - GEE_COUNTY_FIPS_LABEL, - STATE_FIPS_TO_NAME, - output_asset_id, -) -from jdluc.utils.gee import EXPORT_TIMEOUT_S, wait_for_tasks -from jdluc.utils.states import get_multi_state_boundary -from jdluc.utils.version import compute_transform_version - -logger = logging.getLogger(__name__) - -# Heartbeat interval for "still waiting" INFO logs while polling (seconds). -_HEARTBEAT_INTERVAL_S: int = 600 - - -class TransformStage(enum.Enum): - """Public-facing stage label for `TransformError.failed_stage`.""" - - LAND_USE = 'LAND_USE' - EMISSIONS = 'EMISSIONS' - TABLES = 'TABLES' - - -class TransformError(Exception): - """Raised when one of the four region-scope exports fails. - - Carries the failing stage name and the underlying error message so - callers can inspect what went wrong without parsing the exception - string. - """ - - def __init__(self, failed_stage: str, error_message: str) -> None: - self.failed_stage = failed_stage - self.error_message = error_message - super().__init__(f'Transform failed at stage {failed_stage}: {error_message}') - - -@dataclasses.dataclass -class TransformResult: - """Result of a transform-stage run, four region-scope asset IDs.""" - - version: str - region_name: str - land_use_asset_id: str - emissions_asset_id: str - transitions_table_id: str - crops_table_id: str - from_cache: bool - - -def run_transform( - gcp_project: str, - states: list[str], - region_name: str, - force: bool = False, -) -> TransformResult: - """Run the transform stage as four region-scope GEE exports. - - 1. Resolves the region geometry (`get_multi_state_boundary(states)`) - and the counties FeatureCollection (filtered to the region's - STATEFP set, with `county_fips` set to STATEFP+COUNTYFP) once. - 2. Submits and awaits the `land_use` export. - 3. Submits and awaits the `emissions` export (reads the just- - materialized land_use asset). - 4. Submits `transitions` and `crops` table exports in parallel; awaits - both. - - Args: - gcp_project: GCP project ID. Accepted for API parity with - ``extract_all`` / ``run_publish``; the transform stage only - uses the already-initialized Earth Engine session. - states: List of state FIPS codes. Must be non-empty and all in - STATE_FIPS_TO_NAME. - region_name: Region label (e.g. `'delaware'`, `'great_plains_test'`, - `'conus'`). Used in asset names: `{kind}_{region}_{version}`. - force: If True, re-export even if the target asset already exists. - - Returns: - TransformResult with version, region_name, the four asset IDs, and - `from_cache` flag (True iff every export was a cache hit). - - Raises: - ValueError: If `states` is empty or any FIPS is unknown. - TransformError: If any of the four exports fails. - """ - del gcp_project - _validate_inputs(states, region_name) - - version = compute_transform_version() - logger.info(f'Transform version: {version}') - - region_geometry = get_multi_state_boundary(states) - region_bbox = region_geometry.bounds() - fips_band = _load_fips_band_for_states(states) - fips_mask = fips_band.mask() - - any_work = False - - # Stage 1: land_use - land_use_asset_id = output_asset_id('land_use', region_name, version) - try: - land_use_image = build_land_use_image(region_geometry) - task = export_land_use_asset( - image=land_use_image, - region=region_name, - version=version, - geometry=region_geometry, - fips_mask=fips_mask, - force=force, - ) - except Exception as exc: - raise TransformError( - TransformStage.LAND_USE.value, f'{type(exc).__name__}: {exc}' - ) from exc - if task is not None: - any_work = True - try: - _wait_for_export_task( - task, TransformStage.LAND_USE.value, land_use_asset_id - ) - except RuntimeError as exc: - raise TransformError(TransformStage.LAND_USE.value, str(exc)) from exc - - # Stage 2: emissions (reads the materialized land_use asset) - emissions_asset_id = output_asset_id('emissions', region_name, version) - try: - emissions_image = build_emissions_image(land_use_asset_id, region_geometry) - task = export_emissions_asset( - image=emissions_image, - region=region_name, - version=version, - geometry=region_geometry, - fips_mask=fips_mask, - force=force, - ) - except Exception as exc: - raise TransformError( - TransformStage.EMISSIONS.value, f'{type(exc).__name__}: {exc}' - ) from exc - if task is not None: - any_work = True - try: - _wait_for_export_task( - task, TransformStage.EMISSIONS.value, emissions_asset_id - ) - except RuntimeError as exc: - raise TransformError(TransformStage.EMISSIONS.value, str(exc)) from exc - - # Stage 3: transitions + crops in parallel - try: - tables_result = compute_region_tables( - land_use_asset_id=land_use_asset_id, - emissions_asset_id=emissions_asset_id, - region=region_name, - version=version, - fips_band=fips_band, - region_bbox=region_bbox, - force=force, - ) - except Exception as exc: - raise TransformError( - TransformStage.TABLES.value, f'{type(exc).__name__}: {exc}' - ) from exc - - transitions_task = tables_result['transitions_task'] - crops_task = tables_result['crops_task'] - pending_table_tasks: dict[str, Any] = {} - if transitions_task is not None: - any_work = True - pending_table_tasks['transitions'] = transitions_task - if crops_task is not None: - any_work = True - pending_table_tasks['crops'] = crops_task - if pending_table_tasks: - try: - _wait_for_table_tasks( - tasks=pending_table_tasks, - stage_label=TransformStage.TABLES.value, - ) - except RuntimeError as exc: - raise TransformError(TransformStage.TABLES.value, str(exc)) from exc - - return TransformResult( - version=version, - region_name=region_name, - land_use_asset_id=land_use_asset_id, - emissions_asset_id=emissions_asset_id, - transitions_table_id=tables_result['transitions_asset_id'], - crops_table_id=tables_result['crops_asset_id'], - from_cache=not any_work, - ) - - -def _load_fips_band_for_states(state_fips_list: list[str]) -> Any: - """Run-scoped FIPS-valued image: county FIPS in the run's states, masked elsewhere. - - Reads the canonical ``GEE_COUNTY_FIPS_LABEL`` asset (built by - ``extract/county_fips.py``), divides by 1000 to recover the state-FIPS - integer, and ``updateMask``-s to keep only pixels whose state-FIPS is - in ``state_fips_list``. The returned image carries the FIPS values - (1001..56045) on selected pixels and is masked elsewhere. Two - consumers downstream: - - * Raster export sinks call ``image.mask()`` on the result to get a - 0/1 mask suitable for ``updateMask`` against the rasters. - * Summary-table reducers consume the FIPS values directly as one - half of the composite grouping key. - - The FIPS asset is painted at GLAD 30m using paint's pixel-center - rule. - """ - fips_label = ee.Image(GEE_COUNTY_FIPS_LABEL) - state_codes = [int(s) for s in state_fips_list] - state_band = fips_label.divide(1000).toInt() - in_set = state_band.eq(state_codes[0]) - for code in state_codes[1:]: - in_set = in_set.Or(state_band.eq(code)) - return fips_label.updateMask(in_set).rename('county_fips') - - -def _validate_inputs(states: list[str], region_name: str) -> None: - """Reject bad input before submitting any GEE work.""" - if not states: - raise ValueError('states must not be empty') - if not region_name: - raise ValueError('region_name must not be empty') - unknown = [fips for fips in states if fips not in STATE_FIPS_TO_NAME] - if unknown: - raise ValueError( - f'Unknown state FIPS codes (not in STATE_FIPS_TO_NAME): {unknown}' - ) - - -def _wait_for_export_task(task: Any, stage_label: str, asset_id: str) -> None: - """Poll one GEE export task until COMPLETED, raising on FAILED. - - Thin wrapper over ``utils.gee.wait_for_tasks`` that turns the failure - map into a ``RuntimeError`` (which the caller wraps as - ``TransformError(stage, ...)``). - """ - task_id = str(task.id) - failed = wait_for_tasks( - [task_id], - asset_ids_for_logging={task_id: asset_id}, - task_labels={task_id: stage_label}, - heartbeat_interval_s=_HEARTBEAT_INTERVAL_S, - timeout_s=EXPORT_TIMEOUT_S, - ) - if task_id in failed: - raise RuntimeError(f'{stage_label} export failed: {failed[task_id]}') - - -def _wait_for_table_tasks(tasks: dict[str, Any], stage_label: str) -> None: - """Poll multiple GEE export tasks concurrently until all COMPLETED. - - Used for the TABLES stage where transitions and crops can run in - parallel (different table assets, no inter-dependency). Raises - RuntimeError if any task ends FAILED/CANCELLED, naming the first - failed kind. Unlike the prior fail-fast loop, the canonical helper - waits for all tasks to reach a terminal state before returning — - the task itself keeps running on GEE either way, so the only - difference is wall-clock-to-error. - """ - task_id_to_kind = {str(t.id): kind for kind, t in tasks.items()} - failed = wait_for_tasks( - list(task_id_to_kind), - task_labels={ - tid: f'{stage_label}({kind})' for tid, kind in task_id_to_kind.items() - }, - heartbeat_interval_s=_HEARTBEAT_INTERVAL_S, - timeout_s=EXPORT_TIMEOUT_S, - ) - if failed: - first_tid = next(iter(failed)) - kind = task_id_to_kind[first_tid] - raise RuntimeError(f'{stage_label}({kind}) export failed: {failed[first_tid]}') - - -__all__ = [ - 'TransformError', - 'TransformResult', - 'TransformStage', - 'run_transform', -] diff --git a/jdluc/utils.py b/jdluc/utils.py new file mode 100644 index 0000000..5745768 --- /dev/null +++ b/jdluc/utils.py @@ -0,0 +1,74 @@ +import datetime +import functools +import logging +import threading +import typing + +import git +import requests + +logger = logging.getLogger(__name__) + + +@functools.cache +def get_requests_session() -> requests.Session: + logger.info("Creating a fresh session for requests") + return requests.Session() + + +def save_remote_url_to_local_path( + local_path: str, + params: dict[str, str] | str, + remote_url: str, + # 4 MiB + chunk_size: int = 1 << 22, +) -> None: + logger.info(f"GET'ing from {remote_url=:s} with {params=:}") + with get_requests_session().request( + method="GET", params=params, stream=True, url=remote_url + ) as response: + response.raise_for_status() + logger.info(f"Streaming data from {remote_url=:s} to {local_path=:s}") + with open(file=local_path, mode="wb") as fp: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + fp.write(chunk) + + +@functools.cache +def get_git_version(default_branch_name: str = "main") -> str: + repo = git.Repo(__file__, search_parent_directories=True) + if (branch_name := repo.active_branch.name) == default_branch_name: + return f"{branch_name:s}-{repo.head.commit.hexsha[:8]:s}" + else: + return branch_name + + +@functools.cache +def get_git_remote_url(default_remote_name: str = "origin") -> str: + repo = git.Repo(__file__, search_parent_directories=True) + (remote,) = ( + remote for remote in repo.remotes if remote.name == default_remote_name + ) + return next(iter(remote.urls)) + + +def get_utc_timestamp() -> str: + return datetime.datetime.now(datetime.UTC).isoformat() + + +def threadsafe_cache[**P, R](func: typing.Callable[P, R]) -> typing.Callable[P, R]: + cached = functools.cache(func) + + # each cached function has its own lock + lock = threading.Lock() + + @functools.wraps(cached) + def inner(*args: P.args, **kwargs: P.kwargs) -> R: + with lock: + return cached(*args, **kwargs) # type: ignore + + inner.cache_clear = cached.cache_clear # type: ignore + inner.cache_info = cached.cache_info # type: ignore + inner.cache_parameters = cached.cache_parameters # type: ignore + return inner diff --git a/jdluc/utils/__init__.py b/jdluc/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jdluc/utils/_ee_types.py b/jdluc/utils/_ee_types.py deleted file mode 100644 index cab29ee..0000000 --- a/jdluc/utils/_ee_types.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Type aliases for Earth Engine objects. - -The ``ee`` package ships without type stubs, so signatures across the -package use ``Any`` aliases to improve readability. -""" - -from typing import Any - -EEFeature = Any -EEImage = Any -EEFeatureCollection = Any -EEGeometry = Any -EEDictionary = Any -EEList = Any diff --git a/jdluc/utils/asset_management.py b/jdluc/utils/asset_management.py deleted file mode 100644 index 040c739..0000000 --- a/jdluc/utils/asset_management.py +++ /dev/null @@ -1,195 +0,0 @@ -"""GEE asset management helpers. - -Reusable low-level helpers for listing, version-parsing, and safe-deletion of -GEE assets produced by the pipeline. -""" - -import dataclasses -import datetime as dt -import logging -import re -from typing import Any, cast - -import ee - -from jdluc.utils.constants import GEE_ASSET_ROOT - -logger = logging.getLogger(__name__) - -# Version suffix patterns recognized at the end of an asset's terminal segment. -# Anchored at end-of-string; the capture group is the bare version string. -# -# - Transform assets: ``{HEAD[:12]}`` or ``{HEAD[:12]}-dirty-{diff[:8]}`` -# - Publish assets: compound ``{t_sha}_{p_sha}`` tail — the same -# stage-SHA pattern appears twice, separated by an underscore. -# ``parse_compound_version_from_asset_id`` handles that case. -# - Extract: ``v{digits}`` optionally followed by ``_{digits}`` groups -# (e.g. ``v2021``, ``v2017_2020``) -_VERSION_PATTERNS = [ - r'[0-9a-f]{12}-dirty-[0-9a-f]{8}', - r'[0-9a-f]{12}', - r'v\d+(?:_\d+)*', -] -_VERSION_REGEX = re.compile(r'_(' + '|'.join(_VERSION_PATTERNS) + r')$') - -# Publish assets carry two stage SHAs: transform first, publish second. -# Compound regex is end-anchored and greedy-prefers the two-SHA tail. -_STAGE_SHA_PATTERN = r'(?:[0-9a-f]{12}-dirty-[0-9a-f]{8}|[0-9a-f]{12})' -_COMPOUND_VERSION_REGEX = re.compile( - rf'_({_STAGE_SHA_PATTERN})_({_STAGE_SHA_PATTERN})$' -) - - -@dataclasses.dataclass(frozen=True) -class AssetInfo: - """Asset ID + last-update timestamp returned by ``listAssets``. - - ``update_time`` is a timezone-aware UTC ``datetime``; the GEE API - returns RFC3339 strings, parsed once at the source so callers can - compare against ``datetime.now(timezone.utc)`` directly. - """ - - asset_id: str - update_time: dt.datetime - - -def _list_assets_page_iter(prefix: str) -> Any: - """Yield ``ee.data.listAssets`` response dicts, paginated. - - Pulled out as a helper so ``list_assets_matching`` and - ``list_assets_matching_with_metadata`` share one source of truth on - pagination + prefix-matching. - """ - full_prefix = f'{GEE_ASSET_ROOT}/{prefix}' - page_token: str | None = None - while True: - params: dict[str, Any] = {'parent': GEE_ASSET_ROOT} - if page_token: - params['pageToken'] = page_token - # ee.data.listAssets' return type annotation (``dict[str, list[Any]]``) - # is narrower than the real response (which also includes a string - # ``nextPageToken``). Cast once to ``dict[str, Any]`` to avoid a - # per-key type fight. - result = cast(dict[str, Any], ee.data.listAssets(params)) - for asset in result.get('assets') or []: - name = asset.get('name') or asset.get('id') or '' - if name.startswith(full_prefix): - yield asset - next_token = result.get('nextPageToken') - if not next_token: - return - page_token = next_token - - -def list_assets_matching(prefix: str) -> list[str]: - """Return fully-qualified asset IDs under ``GEE_ASSET_ROOT`` whose terminal - segment begins with ``prefix``. - - ``prefix`` is relative to the jdLUC asset root (e.g. - ``land_use_delaware``); the root is prepended internally. Pagination is - handled transparently via ``ee.data.listAssets``' ``nextPageToken``. - """ - return [ - asset.get('name') or asset.get('id') or '' - for asset in _list_assets_page_iter(prefix) - ] - - -def list_assets_matching_with_metadata(prefix: str) -> list[AssetInfo]: - """Same as ``list_assets_matching`` but returns ``AssetInfo``s. - - ``AssetInfo.update_time`` parses the asset's ``updateTime`` field - (RFC3339, e.g. ``'2026-04-23T11:32:00.123456Z'``) into a - timezone-aware UTC ``datetime``. - - Assets without an ``updateTime`` are skipped with a warning — the - GEE API consistently returns it for every asset family we publish, - so its absence indicates an unfamiliar asset shape that the - cleanup tool shouldn't blindly accept. - """ - out: list[AssetInfo] = [] - for asset in _list_assets_page_iter(prefix): - asset_id = asset.get('name') or asset.get('id') or '' - update_time_raw = asset.get('updateTime') - if not update_time_raw: - logger.warning(f'asset {asset_id} has no updateTime; skipping') - continue - try: - update_time = _parse_rfc3339_utc(update_time_raw) - except ValueError as exc: - logger.warning( - f'asset {asset_id} has unparseable updateTime ' - f'{update_time_raw!r}: {exc}; skipping' - ) - continue - out.append(AssetInfo(asset_id=asset_id, update_time=update_time)) - return out - - -def _parse_rfc3339_utc(text: str) -> dt.datetime: - """Parse an RFC3339 timestamp into a tz-aware UTC ``datetime``. - - GEE returns timestamps with a ``Z`` suffix (UTC); ``fromisoformat`` - in Python 3.11 accepts the ``Z`` form on most builds but not all, - so we normalize to ``+00:00`` first. - """ - normalized = text.replace('Z', '+00:00') if text.endswith('Z') else text - parsed = dt.datetime.fromisoformat(normalized) - if parsed.tzinfo is None: - # Defensive: an RFC3339 timestamp is always tz-aware, but if - # ``fromisoformat`` returns a naïve dt for any reason, treat it - # as UTC explicitly. - parsed = parsed.replace(tzinfo=dt.UTC) - return parsed.astimezone(dt.UTC) - - -def parse_version_from_asset_id(asset_id: str) -> str | None: - """Extract the single trailing version suffix from an asset ID. - - Recognizes the transform-stage SHA forms - (``[0-9a-f]{12}`` optionally followed by ``-dirty-[0-9a-f]{8}``) and the - extract-stage ``v{digits}[_{digits}…]`` form. Anything else returns - ``None``. Publish assets carry a compound ``{t_sha}_{p_sha}`` tail — - use ``parse_compound_version_from_asset_id`` for those. - """ - terminal = asset_id.rsplit('/', 1)[-1] - match = _VERSION_REGEX.search(terminal) - if match is None: - return None - return match.group(1) - - -def parse_compound_version_from_asset_id( - asset_id: str, -) -> tuple[str, str] | None: - """Extract ``(transform_sha, publish_sha)`` from a publish-asset ID. - - Publish assets (BigQuery tables) use a compound - ``{prefix}_{region}_{t_sha}_{p_sha}`` naming scheme. Returns the pair - when both stage SHAs match (either clean or dirty form each); returns - ``None`` if the terminal segment only has one trailing SHA - (i.e. a transform-stage asset rather than a publish-stage asset). - """ - terminal = asset_id.rsplit('/', 1)[-1] - match = _COMPOUND_VERSION_REGEX.search(terminal) - if match is None: - return None - return match.group(1), match.group(2) - - -def delete_asset_safely(asset_id: str, dry_run: bool = False) -> bool: - """Delete a GEE asset, logging and returning ``False`` on failure. - - When ``dry_run`` is true, logs what would be deleted and returns ``True`` - without touching GEE. - """ - if dry_run: - logger.info(f'[dry-run] would delete {asset_id}') - return True - try: - ee.data.deleteAsset(asset_id) - logger.info(f'deleted {asset_id}') - return True - except ee.EEException as exc: - logger.error(f'failed to delete {asset_id}: {exc}') - return False diff --git a/jdluc/utils/constants.py b/jdluc/utils/constants.py deleted file mode 100644 index 2c6b8de..0000000 --- a/jdluc/utils/constants.py +++ /dev/null @@ -1,825 +0,0 @@ -"""Cross-module constants for the jdLUC pipeline. - -Central home for expected versions for input datasets, simplified land use category codes, -IPCC emissions factor tables, crop group definitions, and so on. - -See specs/pipeline_tech_design.md § constants.py for details. -""" - -from itertools import product -from typing import Literal, NamedTuple - -from jdluc.utils.transitions import encode_transition - -# =========================================================================== -# DEPLOYMENT CONFIGURATION — edit these six values to point jdLUC at your GCP -# =========================================================================== -# Every GCP / GCS / BQ / GEE identifier the pipeline, and scripts reach for -# is defined here. To deploy against your GCP, edit the values below directly. -# Every consumer imports from ``utils.constants``, so there are no other call -# sites to chase. - -# GCP project for application-default credentials, GEE init, and BQ clients. -GCP_PROJECT: str = 'cornerstone-data' - -# BigQuery destination for the published `transitions` and `crops` tables. -# ``BQ_JOB_LOCATION`` must match the dataset's region — ``ee.batch.Export. -# table.toBigQuery`` targets it explicitly. -BQ_PROJECT: str = GCP_PROJECT -BQ_DATASET: str = 'jdluc' -BQ_JOB_LOCATION: str = 'US' - -# GCS bucket the extract stage uses to mirror non-native source-file fetches -# (Harris AGB, Huang BGB, NASS yields, IPCC climate zones, GFW peatlands). -GCS_BUCKET_NAME: str = 'cornerstone-luc' - -# Parent GEE folder for every jdLUC-owned asset (extract outputs, transform -# outputs, publish-stage tabular exports). -GEE_ASSET_ROOT: str = f'projects/{BQ_PROJECT:s}/assets/cornerstone-luc' - -# =========================================================================== -# END deployment configuration -# =========================================================================== - - -class LandUseCategory(NamedTuple): - code: int - glad_values: list[int] - - -# --------------------------------------------------------------------------- -# CONUS state FIPS <-> canonical region name -# --------------------------------------------------------------------------- -# Every per-state transform asset is named using the canonical state name -# (e.g. land_use_iowa_{version}) rather than the FIPS code, so the same -# per-state asset is reused across any region that includes that state. -# Names are snake_case lowercase matching TIGER's NAME -# field. Alaska (02) and Hawaii (15) are excluded — this methodology is -# CONUS-scoped. - -STATE_FIPS_TO_NAME: dict[str, str] = { - '01': 'alabama', - '04': 'arizona', - '05': 'arkansas', - '06': 'california', - '08': 'colorado', - '09': 'connecticut', - '10': 'delaware', - '11': 'district_of_columbia', - '12': 'florida', - '13': 'georgia', - '16': 'idaho', - '17': 'illinois', - '18': 'indiana', - '19': 'iowa', - '20': 'kansas', - '21': 'kentucky', - '22': 'louisiana', - '23': 'maine', - '24': 'maryland', - '25': 'massachusetts', - '26': 'michigan', - '27': 'minnesota', - '28': 'mississippi', - '29': 'missouri', - '30': 'montana', - '31': 'nebraska', - '32': 'nevada', - '33': 'new_hampshire', - '34': 'new_jersey', - '35': 'new_mexico', - '36': 'new_york', - '37': 'north_carolina', - '38': 'north_dakota', - '39': 'ohio', - '40': 'oklahoma', - '41': 'oregon', - '42': 'pennsylvania', - '44': 'rhode_island', - '45': 'south_carolina', - '46': 'south_dakota', - '47': 'tennessee', - '48': 'texas', - '49': 'utah', - '50': 'vermont', - '51': 'virginia', - '53': 'washington', - '54': 'west_virginia', - '55': 'wisconsin', - '56': 'wyoming', -} -STATE_NAME_TO_FIPS: dict[str, str] = { - name: fips for fips, name in STATE_FIPS_TO_NAME.items() -} -CONUS_STATE_FIPS: list[str] = sorted(STATE_FIPS_TO_NAME) - - -# --------------------------------------------------------------------------- -# GEE asset paths -# --------------------------------------------------------------------------- - -# Root folder for all jdLUC-owned GEE assets (extract, transform, publish). -# Per-asset IDs are built as ``f'{GEE_ASSET_ROOT}/{asset_name}'``. -# ``GEE_ASSET_ROOT`` itself is defined in the deployment configuration block -# at the top of this module. - -# US state boundaries (TIGER/Census 2018). -GEE_TIGER_STATES = 'TIGER/2018/States' - -# USDA Cropland Data Layer (annual). 30 m, native Albers grid. -GEE_CDL_COLLECTION = 'USDA/NASS/CDL' - -# GLAD cropland binary, by-year (Potapov et al. 2022). Asset IDs are formed -# as f'{GEE_GLAD_CROPLAND_PREFIX}{year}' for the supported epochs (see -# GLAD_CROPLAND_EPOCHS). Used by analyses backing scripts that compare -# CDL and GLAD cropland directly. -GEE_GLAD_CROPLAND_PREFIX = 'users/potapovpeter/Global_cropland_' -GLAD_CROPLAND_EPOCHS: list[int] = [2003, 2007, 2011, 2015, 2019] - -# Canonical asset-ID pattern for every transform/publish output raster and -# table: ``{GEE_ASSET_ROOT}/{name}_{region}_{version}``. See -# specs/pipeline_tech_design.md § Outputs. -OutputName = Literal['land_use', 'emissions', 'transitions', 'crops'] - - -def output_asset_id(name: OutputName, region: str, version: str) -> str: - """Canonical GEE asset ID for a transform/publish output.""" - return f'{GEE_ASSET_ROOT}/{name}_{region}_{version}' - - -# --------------------------------------------------------------------------- -# Canonical raster grid (GLAD GLCLUC v2 native) -# --------------------------------------------------------------------------- -# Every raster computation and export in the pipeline is pinned to this grid. -# See specs/pipeline_tech_design.md § Platform. -GLAD_CRS = 'EPSG:4326' -GLAD_CRS_TRANSFORM: list[float] = [0.00025, 0, -180, 0, -0.00025, 90] - -# --------------------------------------------------------------------------- -# GLAD GLCLUC v2 epoch years -# --------------------------------------------------------------------------- -GLAD_EPOCH_YEARS: list[int] = [2000, 2005, 2010, 2015, 2020] -GLAD_EPOCH_PAIRS: list[tuple[int, int]] = [ - (GLAD_EPOCH_YEARS[i], GLAD_EPOCH_YEARS[i + 1]) - for i in range(len(GLAD_EPOCH_YEARS) - 1) -] - -# --------------------------------------------------------------------------- -# Land use categories and GLAD GLC remap -# --------------------------------------------------------------------------- -# Each entry maps our category name to its integer code and the GLAD GLCLUC v2 -# raw pixel values that map to it. 0 is reserved for "no transition" in the -# encoded transition bands; it is never a valid land use category. -# GLAD value ranges from specs/methodology.md § "Land cover transitions". - -LAND_USE_CATEGORIES: dict[str, LandUseCategory] = { - 'forest': LandUseCategory(1, list(range(25, 49))), - 'wetland_forest': LandUseCategory(2, list(range(125, 149))), - 'short_vegetation': LandUseCategory(3, list(range(1, 25))), - 'wetland_short_vegetation': LandUseCategory(4, list(range(100, 125))), - 'cropland': LandUseCategory(5, [244]), - 'built_up': LandUseCategory(6, [250]), - 'water': LandUseCategory(7, list(range(200, 208))), - 'snow_ice': LandUseCategory(8, [241]), - 'bare': LandUseCategory(9, [0]), -} - -# --------------------------------------------------------------------------- -# Extract-side: upstream-dataset versions, source URLs, pinned snapshots -# --------------------------------------------------------------------------- -# Expected versions for the four non-native datasets the extract stage -# ingests into GEE. Asset IDs are composed as -# ``{GEE_ASSET_ROOT}/{family}_{version}`` so a version bump forces a cache -# miss and re-extract. - -EXPECTED_HARRIS_VERSION: str = 'v2021' # Harris et al. (2021) publication year. -EXPECTED_HUANG_VERSION: str = 'v2021' # Huang et al. (2021). -EXPECTED_IPCC_VERSION: str = 'v2006' # Ogle et al. (2006). -EXPECTED_GFW_PEATLANDS_VERSION: str = 'v20230315' # GFW Global Peatlands publication. -EXPECTED_TIGER_COUNTIES_VERSION: str = 'v2018' # TIGER/2018/Counties vintage. - -# Harris AGB tile URLs live on GFW's ArcGIS FeatureServer; `Mg_ha_1_download` -# is the per-feature signed URL. 21 tiles cover CONUS (rows 30N/40N/50N × -# columns 070W–130W). -HARRIS_AGB_ARCGIS_FEATURESERVER: str = ( - 'https://services2.arcgis.com/g8WusZB13b9OegfU/arcgis/rest/services/' - 'Aboveground_Live_Woody_Biomass_Density/FeatureServer/0/query' -) - -# Huang BGB NetCDF lives at Figshare DOI 10.6084/m9.figshare.12199637.v1. -HUANG_BGB_FIGSHARE_URL: str = 'https://ndownloader.figshare.com/files/22432460' - -# USDA NASS QuickStats crop archive. The URL is date-stamped per release; we -# pin to a specific snapshot so the extract version string is coupled to a -# concrete upstream file. Bumping the date requires adding an entry to -# ``_NASS_RELEASE_DATE_TO_VERSION`` below — module import fails otherwise. -NASS_QUICKSTATS_URL_TEMPLATE: str = ( - 'https://www.nass.usda.gov/datasets/qs.crops_{date}.txt.gz' -) -NASS_QUICKSTATS_RELEASE_DATE: str = '20260508' - -# Registered (release_date → version) mappings. Version strings encode the -# ``NASS_YIELD_YEARS`` window the transform averages over — keeping the tag -# window-aligned rather than release-aligned keeps the cache semantically -# meaningful even when NASS reissues the same year's data under a new -# release date. -_NASS_RELEASE_DATE_TO_VERSION: dict[str, str] = { - '20260228': 'v2017_2020', - '20260423': 'v2017_2020', - '20260424': 'v2017_2020', - '20260508': 'v2017_2020', -} - - -def _derive_nass_version_from_release_date(release_date: str) -> str: - """Mechanical (release_date → version) mapping. - - Import fails if ``NASS_QUICKSTATS_RELEASE_DATE`` is bumped without - registering the corresponding version. This prevents silent staleness - where a new release reuses the old asset ID and bypasses the extract - cache-miss check. - """ - try: - return _NASS_RELEASE_DATE_TO_VERSION[release_date] - except KeyError as exc: - raise ValueError( - f'NASS release date {release_date!r} has no registered version. ' - f'Add an entry to _NASS_RELEASE_DATE_TO_VERSION in ' - f'utils/constants.py when bumping NASS_QUICKSTATS_RELEASE_DATE.' - ) from exc - - -EXPECTED_NASS_VERSION: str = _derive_nass_version_from_release_date( - NASS_QUICKSTATS_RELEASE_DATE -) - -# Zenodo record 7303808 hosts the Ogle et al. (2006) IPCC climate zone -# raster as a single GeoTIFF. The extract module resolves the file -# download URL through Zenodo's record API — the record-level URL below -# is the stable identifier. -IPCC_CLIMATE_ZONES_ZENODO_URL: str = 'https://zenodo.org/api/records/7303808' - -# GFW Global Peatlands raster composite (CC BY 4.0): Xu et al. 2018 -# (PEATMAP) above 40°N + Gumbricht et al. 2017 below 40°N + regional -# overrides (Crezee 2022 Congo, Hastie 2022 lowland Peru, Miettinen -# 2016 Indonesia/Malaysia), all rasterized to 30 m to align with the -# Hansen Global Forest Change grid. Data is distributed as 10°×10° -# GeoTIFF tiles via GFW's data API; the URL template below yields a -# direct GeoTIFF download per ``tile_id``. ``pixel_meaning=is`` selects -# the binary peatland-presence band. -# -# The API key ``GFW_DATA_API_KEY`` ships embedded in any user's CSV -# download from https://data.globalforestwatch.org/datasets/gfw::global-peatlands — -# it's a public rate-limited token, not a secret. If the key 401s -# (analogous to NASS release-date rotation), re-download the CSV -# from the GFW Open Data Portal and bump this constant. -GFW_DATA_API_KEY: str = '2d60cd88-8348-4c0f-a6d5-bd9adb585a8c' -GFW_PEATLANDS_URL_TEMPLATE: str = ( - 'https://data-api.globalforestwatch.org/dataset/gfw_peatlands/' - 'v20230315/download/geotiff?grid=10/40000&tile_id={tile_id}' - '&pixel_meaning=is&x-api-key={api_key}' -) - -# --------------------------------------------------------------------------- -# Transform-stage input asset IDs (jdLUC-owned extracts + GEE-native sources) -# --------------------------------------------------------------------------- -GEE_HARRIS_AGB: str = f'{GEE_ASSET_ROOT}/harris_agb_conus_{EXPECTED_HARRIS_VERSION}' -GEE_HUANG_BGB: str = f'{GEE_ASSET_ROOT}/huang_bgb_conus_{EXPECTED_HUANG_VERSION}' -GEE_SOILGRIDS_SOC: str = 'projects/soilgrids-isric/ocs_mean' -GEE_IPCC_CLIMATE_ZONES: str = ( - f'{GEE_ASSET_ROOT}/ipcc_climate_zones_{EXPECTED_IPCC_VERSION}' -) -GEE_GFW_PEATLANDS: str = ( - f'{GEE_ASSET_ROOT}/gfw_peatlands_{EXPECTED_GFW_PEATLANDS_VERSION}' -) -# CONUS county-FIPS label raster — single-band int32 (FIPS = state_fips × 1000 -# + county_fips, range 1001–56045) painted from TIGER counties at GLAD 30m. -# Pixels outside any CONUS county are masked. This asset is the single -# source of geographic truth for the transform pipeline, doubling as -# polygon mask + per-county grouping band. -GEE_COUNTY_FIPS_LABEL: str = ( - f'{GEE_ASSET_ROOT}/county_fips_conus_{EXPECTED_TIGER_COUNTIES_VERSION}' -) - -# --------------------------------------------------------------------------- -# Dataset inventory -# --------------------------------------------------------------------------- -# Non-native datasets (those ingested by the extract stage) additionally -# carry ``expected_version`` and ``source_url`` — the orchestrator reads -# both when deciding whether to re-extract. Native datasets omit them. -DATASET_INVENTORY: dict[str, dict[str, str]] = { - 'glad_glcluc_v2': { - 'gee_asset_id': 'projects/glad/GLCLU2020/v2/LCLUC_{year}', - 'resolution_hint': '0.00025deg (~30m)', - 'native_or_extracted': 'native', - }, - 'usda_cdl': { - 'gee_asset_id': GEE_CDL_COLLECTION, - 'resolution_hint': '30m (native Albers)', - 'native_or_extracted': 'native', - }, - 'gfw_peatlands': { - 'gee_asset_id': GEE_GFW_PEATLANDS, - 'resolution_hint': '0.00025deg (~30m), GLAD grid', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_GFW_PEATLANDS_VERSION, - # source_url is the URL template (api-key elided); per-tile URLs - # are formatted at extract time. Same posture as - # HARRIS_AGB_ARCGIS_FEATURESERVER's role for harris_agb. - 'source_url': GFW_PEATLANDS_URL_TEMPLATE.replace('{api_key}', ''), - }, - 'tiger_states': { - 'gee_asset_id': GEE_TIGER_STATES, - 'resolution_hint': 'vector', - 'native_or_extracted': 'native', - }, - 'county_fips': { - 'gee_asset_id': GEE_COUNTY_FIPS_LABEL, - 'resolution_hint': '0.00025deg (~30m), GLAD grid', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_TIGER_COUNTIES_VERSION, - # Source is the GEE-native TIGER counties FeatureCollection; - # source_url documents the TIGER 2018 vintage upstream. - 'source_url': ('https://www2.census.gov/geo/tiger/TIGER2018/COUNTY/'), - }, - 'harris_agb': { - 'gee_asset_id': GEE_HARRIS_AGB, - 'resolution_hint': '0.00025deg (~30m), GLAD grid', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_HARRIS_VERSION, - 'source_url': HARRIS_AGB_ARCGIS_FEATURESERVER, - }, - 'huang_bgb': { - 'gee_asset_id': GEE_HUANG_BGB, - 'resolution_hint': '0.0083deg (~1km)', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_HUANG_VERSION, - 'source_url': HUANG_BGB_FIGSHARE_URL, - }, - 'soilgrids_soc': { - 'gee_asset_id': GEE_SOILGRIDS_SOC, - 'resolution_hint': '250m (native Interrupted Goode Homolosine)', - 'native_or_extracted': 'native', - }, - 'ipcc_climate_zones': { - 'gee_asset_id': GEE_IPCC_CLIMATE_ZONES, - 'resolution_hint': '0.00025deg (~30m), GLAD grid (upsampled from 0.5deg)', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_IPCC_VERSION, - 'source_url': IPCC_CLIMATE_ZONES_ZENODO_URL, - }, -} - -# Dataset-family keys the extract orchestrator iterates. Order reflects -# extract wall-clock cost (county_fips is smallest — server-side paint with no -# HTTP/GCS round-trip; harris is largest); the orchestrator is sequential and -# this keeps fast-failing mis-configurations surfacing first. -NON_NATIVE_DATASETS: list[str] = [ - 'county_fips', - 'ipcc_climate_zones', - 'gfw_peatlands', - 'huang_bgb', - 'nass_yields', - 'harris_agb', -] - -# --------------------------------------------------------------------------- -# Emissive LUC transition vocabulary -# --------------------------------------------------------------------------- -# The 9-pair emissive LUC vocabulary drives the transform-stage vegetation + -# SOC LUC emission paths and the `transitions` summary table. The vocabulary is -# defined as (short_name, from_category_key, to_category_key) tuples so that -# downstream derivations — EMISSIONS_TYPE_NAMES, EMISSIVE_LUC_PAIRS, -# EMISSIVE_ENCODED_CODES, PEATLAND_DRAINAGE_PAIRS — are all keyed off a single -# source of truth. Category short names in EMISSIONS_TYPE_NAMES follow the -# methodology spec (`short_veg`, not `short_vegetation`). -# See specs/methodology.md § Land cover transitions and -# specs/pipeline_tech_design.md § Rasters. - -_EMISSIVE_TRANSITION_CATEGORIES: list[tuple[str, str, str]] = [ - ('forest_to_cropland', 'forest', 'cropland'), - ('forest_to_built_up', 'forest', 'built_up'), - ('forest_to_short_veg', 'forest', 'short_vegetation'), - ('wetland_forest_to_cropland', 'wetland_forest', 'cropland'), - ('wetland_forest_to_built_up', 'wetland_forest', 'built_up'), - ('short_veg_to_cropland', 'short_vegetation', 'cropland'), - ('short_veg_to_built_up', 'short_vegetation', 'built_up'), - ('wetland_short_veg_to_cropland', 'wetland_short_vegetation', 'cropland'), - ('wetland_short_veg_to_built_up', 'wetland_short_vegetation', 'built_up'), -] - -EMISSIVE_LUC_PAIRS: list[tuple[int, int]] = [ - (LAND_USE_CATEGORIES[f].code, LAND_USE_CATEGORIES[t].code) - for (_, f, t) in _EMISSIVE_TRANSITION_CATEGORIES -] -EMISSIVE_ENCODED_CODES: list[int] = [ - encode_transition(f, t) for (f, t) in EMISSIVE_LUC_PAIRS -] - -# Peatland conversion emissions are driven by drainage, which is imposed by -# cropland/built_up destinations — not by forest→short_veg transitions alone. -_PEATLAND_DRAINAGE_DEST_CODES: frozenset[int] = frozenset( - {LAND_USE_CATEGORIES['cropland'].code, LAND_USE_CATEGORIES['built_up'].code} -) -PEATLAND_DRAINAGE_PAIRS: list[tuple[int, int]] = [ - (f, t) for (f, t) in EMISSIVE_LUC_PAIRS if t in _PEATLAND_DRAINAGE_DEST_CODES -] -PEATLAND_DRAINAGE_ENCODED_CODES: list[int] = [ - encode_transition(f, t) for (f, t) in PEATLAND_DRAINAGE_PAIRS -] - -# Full 11-entry vocabulary: 9 emissive LUC pairs + 2 peatland band categories. -# transform/summary_tables.py reduces emissions bands keyed off these names. -EMISSIONS_TYPE_NAMES: list[str] = [ - name for (name, _, _) in _EMISSIVE_TRANSITION_CATEGORIES -] + ['peatland_conversion', 'peatland_occupation'] - -# --------------------------------------------------------------------------- -# IPCC climate zones (10-zone canonical vocabulary) -# --------------------------------------------------------------------------- -# Codes 1–10 follow the ordering of the Houghton/BLUE grassland vegetation-C -# table in specs/methodology.md § Grassland and shrubland. The Ogle et al. -# Zenodo raster (DOI 10.5281/zenodo.7303808) uses a different integer code -# assignment; IPCC_CLIMATE_ZONE_NATIVE_REMAP rewrites native codes into the -# canonical ones at read time (in transform/emissions.py::_load_ipcc_climate). -# Polar zones (Ogle native 11, 12) fall through to 0 (out-of-vocabulary; -# methodology's Houghton table has no polar row, and CONUS has no polar -# pixels). - -IPCC_CLIMATE_ZONES: dict[str, int] = { - 'tropical_wet': 1, - 'tropical_moist': 2, - 'tropical_dry': 3, - 'tropical_montane': 4, - 'warm_temperate_moist': 5, - 'warm_temperate_dry': 6, - 'cool_temperate_moist': 7, - 'cool_temperate_dry': 8, - 'boreal_moist': 9, - 'boreal_dry': 10, -} -IPCC_CLIMATE_ZONE_NAMES: dict[int, str] = { - code: name for name, code in IPCC_CLIMATE_ZONES.items() -} - -# Ogle native code → canonical code. Derived from -# ipcc_climate_zones_2019.R in the Zenodo record. -IPCC_CLIMATE_ZONE_NATIVE_REMAP: dict[int, int] = { - 1: IPCC_CLIMATE_ZONES['tropical_montane'], - 2: IPCC_CLIMATE_ZONES['tropical_wet'], - 3: IPCC_CLIMATE_ZONES['tropical_moist'], - 4: IPCC_CLIMATE_ZONES['tropical_dry'], - 5: IPCC_CLIMATE_ZONES['warm_temperate_moist'], - 6: IPCC_CLIMATE_ZONES['warm_temperate_dry'], - 7: IPCC_CLIMATE_ZONES['cool_temperate_moist'], - 8: IPCC_CLIMATE_ZONES['cool_temperate_dry'], - 9: IPCC_CLIMATE_ZONES['boreal_moist'], - 10: IPCC_CLIMATE_ZONES['boreal_dry'], - # Ogle native 11 (polar_moist) and 12 (polar_dry) have no methodology - # analog and fall through to 0 via defaultValue in .remap(). -} - -# --------------------------------------------------------------------------- -# Vegetation carbon tables -# --------------------------------------------------------------------------- -# Houghton/BLUE total vegetation carbon density (tC/ha) for grassland / -# shrubland pixels, keyed by IPCC climate zone. See specs/methodology.md -# § Grassland and shrubland. -GRASSLAND_VEGETATION_TC_HA_BY_ZONE: dict[int, float] = { - IPCC_CLIMATE_ZONES['tropical_wet']: 18.0, - IPCC_CLIMATE_ZONES['tropical_moist']: 18.0, - IPCC_CLIMATE_ZONES['tropical_dry']: 7.0, - IPCC_CLIMATE_ZONES['tropical_montane']: 7.0, - IPCC_CLIMATE_ZONES['warm_temperate_moist']: 7.0, - IPCC_CLIMATE_ZONES['warm_temperate_dry']: 5.0, - IPCC_CLIMATE_ZONES['cool_temperate_moist']: 7.0, - IPCC_CLIMATE_ZONES['cool_temperate_dry']: 5.0, - IPCC_CLIMATE_ZONES['boreal_moist']: 6.0, - IPCC_CLIMATE_ZONES['boreal_dry']: 3.0, -} - -# CDM AR-TOOL-12 dead wood and litter factors, expressed as fractions of -# above-ground biomass. See specs/methodology.md § Dead organic matter. The -# methodology's 5-row table collapses to the 10 IPCC zones below: the -# tropical rows split across zones 1–4; temperate and boreal share the same -# (0.08, 0.04) row per CDM AR-TOOL-12. -DOM_FACTORS_BY_ZONE: dict[int, tuple[float, float]] = { - IPCC_CLIMATE_ZONES['tropical_wet']: (0.06, 0.01), - IPCC_CLIMATE_ZONES['tropical_moist']: (0.01, 0.01), - IPCC_CLIMATE_ZONES['tropical_dry']: (0.02, 0.04), - IPCC_CLIMATE_ZONES['tropical_montane']: (0.07, 0.01), - IPCC_CLIMATE_ZONES['warm_temperate_moist']: (0.08, 0.04), - IPCC_CLIMATE_ZONES['warm_temperate_dry']: (0.08, 0.04), - IPCC_CLIMATE_ZONES['cool_temperate_moist']: (0.08, 0.04), - IPCC_CLIMATE_ZONES['cool_temperate_dry']: (0.08, 0.04), - IPCC_CLIMATE_ZONES['boreal_moist']: (0.08, 0.04), - IPCC_CLIMATE_ZONES['boreal_dry']: (0.08, 0.04), -} - -# --------------------------------------------------------------------------- -# Carbon fractions and CO₂/C ratio -# --------------------------------------------------------------------------- -# IPCC 2006, Vol 4, Ch 4, §4.5 (living woody biomass). -CARBON_FRACTION_LIVE: float = 0.47 -# CDM AR-TOOL-12 (dead wood and litter pools). -CARBON_FRACTION_DEAD_WOOD: float = 0.50 -CARBON_FRACTION_LITTER: float = 0.37 -CO2_C_RATIO: float = 44.0 / 12.0 - -# Global forest mean root-to-shoot ratio from Huang et al. (2021), -# Earth System Science Data 13:4263-4274. Used only for Huang BGB NoData -# gap-fill. -ROOT_SHOOT_RATIO_TEMPERATE: float = 0.25 - -# IPCC default reference SOC stock for warm temperate moist mineral soil, -# low-activity clay (IPCC 2019 Refinement, Vol 4, Table 2.3). Used as the -# SoilGrids NoData gap-fill value. -IPCC_REFERENCE_SOC_STOCK_TC_HA: float = 63.0 - -# --------------------------------------------------------------------------- -# IPCC SOC stock-change factors (IPCC 2019, Vol 4, Ch 5, Table 5.5) -# --------------------------------------------------------------------------- -# SOC loss fraction per emissive (from, to) transition, keyed by IPCC climate -# zone. IPCC 2019 Tier 1 defines a single F_LU per climate regime for -# long-term cultivated cropland (Table 5.5), applied identically regardless -# of source land use: native forest and native grassland are both at F_LU = 1 -# (reference) per Table 5.10. Management and input factors default to 1 -# (full tillage, nominal input), so the loss fraction is simply 1 - F_LU. -# forest→short_veg has loss 0 (both at reference). -# See specs/methodology.md § Soil organic carbon for mineral soils. - -_F_LU_CROPLAND_BY_ZONE: dict[int, float] = { - # zone_code → F_LU for long-term cultivated cropland (IPCC 2019 Vol 4 - # Table 5.5). Same factor applies to forest-family and short-veg-family - # sources, and to both cropland and built_up destinations. - IPCC_CLIMATE_ZONES['tropical_wet']: 0.83, # Tropical Moist/Wet - IPCC_CLIMATE_ZONES['tropical_moist']: 0.83, # Tropical Moist/Wet - IPCC_CLIMATE_ZONES['tropical_dry']: 0.92, # Tropical Dry - # Tropical montane has no explicit F_LU row in Table 5.5; per footnote 4, - # montane factors are approximated as the mean of temperate and tropical - # stock changes. Using the mean of Warm Temperate Moist (0.69) and - # Tropical Moist/Wet (0.83) = 0.76. - IPCC_CLIMATE_ZONES['tropical_montane']: 0.76, - IPCC_CLIMATE_ZONES['warm_temperate_moist']: 0.69, - IPCC_CLIMATE_ZONES['warm_temperate_dry']: 0.76, - # Cool Temperate and Boreal share rows in Table 5.5. - IPCC_CLIMATE_ZONES['cool_temperate_moist']: 0.70, - IPCC_CLIMATE_ZONES['cool_temperate_dry']: 0.77, - IPCC_CLIMATE_ZONES['boreal_moist']: 0.70, - IPCC_CLIMATE_ZONES['boreal_dry']: 0.77, -} - - -def _build_soc_loss_fractions() -> dict[tuple[int, int, int], float]: - code = {name: cat.code for name, cat in LAND_USE_CATEGORIES.items()} - forest_family = ('forest', 'wetland_forest') - short_veg_family = ('short_vegetation', 'wetland_short_vegetation') - developed = ('cropland', 'built_up') - result: dict[tuple[int, int, int], float] = {} - for zone, f_lu in _F_LU_CROPLAND_BY_ZONE.items(): - loss = 1.0 - f_lu - for src, dst in product(forest_family + short_veg_family, developed): - result[(zone, code[src], code[dst])] = loss - # forest → short_vegetation is emissive for vegetation C, but SOC - # loss is 0 (both are reference conditions in IPCC Table 5.10). - result[(zone, code['forest'], code['short_vegetation'])] = 0.0 - return result - - -IPCC_SOC_LOSS_FRACTIONS: dict[tuple[int, int, int], float] = _build_soc_loss_fractions() - -# --------------------------------------------------------------------------- -# Peatland parameters (methodology § Peatland emissions, supplement § GHGP -# parameterization) -# --------------------------------------------------------------------------- -PEATLAND_P_LUC_TCO2_HA: float = 621.0 -PEATLAND_E_LM_TCO2E_HA_YR: float = 37.3 - -# --------------------------------------------------------------------------- -# GHGP 2020-epoch allocation weights -# --------------------------------------------------------------------------- -# Linear discount weights applied to per-epoch LUC and peatland-conversion -# emissions when summing into the 2020 allocated band. Keyed by epoch -# (from_year, to_year) tuple so the mapping is immune to reordering of -# GLAD_EPOCH_PAIRS. See specs/methodology.md § "Allocating emissions to crop -# years". -GHGP_EPOCH_WEIGHTS_2020: dict[tuple[int, int], float] = { - (2000, 2005): 0.0125, - (2005, 2010): 0.0375, - (2010, 2015): 0.0625, - (2015, 2020): 0.0875, -} - -# --------------------------------------------------------------------------- -# Per-epoch band-name helpers -# --------------------------------------------------------------------------- -# Centralized so a typo anywhere is detected at import time rather than -# silently producing a band the consumer never reads. - - -def transitions_band(from_year: int, to_year: int) -> str: - """Land_use band name for the (from_year, to_year) transition epoch.""" - return f'transitions_{from_year}_{to_year}' - - -def luc_emissions_band(from_year: int, to_year: int) -> str: - """Emissions band: per-epoch land-use-change CO2e (tCO2/ha).""" - return f'luc_emissions_{from_year}_{to_year}' - - -def peatland_conversion_band(from_year: int, to_year: int) -> str: - """Emissions band: per-epoch peatland conversion CO2 (tCO2/ha).""" - return f'peatland_conversion_{from_year}_{to_year}' - - -# --------------------------------------------------------------------------- -# emissions raster band names (11 bands, spec order) -# --------------------------------------------------------------------------- -# See specs/pipeline_tech_design.md § Rasters for the full band definition. -EMISSIONS_BAND_NAMES: list[str] = ( - [luc_emissions_band(a, b) for (a, b) in GLAD_EPOCH_PAIRS] - + [peatland_conversion_band(a, b) for (a, b) in GLAD_EPOCH_PAIRS] - + [ - 'peatland_occupation_2020', - 'allocated_luc_emissions_2020', - 'allocated_peatland_emissions_2020', - ] -) - -# --------------------------------------------------------------------------- -# emissions_type code vocabulary -# --------------------------------------------------------------------------- -# Integer codes for the 11-entry EMISSIONS_TYPE_NAMES vocabulary, used by -# transform/summary_tables.py as the group key for per-county reduceRegions. -# Codes are 1-indexed so that 0 can remain the "non-emissive" sentinel that -# gets masked out of each reducer's input. -EMISSIONS_TYPE_CODES: dict[str, int] = { - name: code for code, name in enumerate(EMISSIONS_TYPE_NAMES, start=1) -} -EMISSIONS_TYPE_CODE_TO_NAME: dict[int, str] = { - code: name for name, code in EMISSIONS_TYPE_CODES.items() -} - -# --------------------------------------------------------------------------- -# Epoch transition labels -# --------------------------------------------------------------------------- -# Canonical string labels for the four GLAD epoch windows, used as the -# `epoch_transition` column value in the `transitions` summary table rows for -# LUC and peatland_conversion. Peatland_occupation rows use a distinct -# single-year label because occupation is an annual emission, not a window. -EPOCH_TRANSITION_LABELS: list[str] = [f'{a}_{b}' for (a, b) in GLAD_EPOCH_PAIRS] -PEATLAND_OCCUPATION_EPOCH_LABEL: str = '2020' - -# --------------------------------------------------------------------------- -# Row crop vocabulary (methodology § Allocating emissions to crops) -# --------------------------------------------------------------------------- -# CDL crop codes grouped by the three crop families the methodology -# emissions-factors against. Double-crop codes (26, 225, 236, 238, 240, 254) -# are intentionally excluded and therefore flow through neither the emissions -# numerator nor the production denominator — a known issue, tracked in -# specs/methodology.md § Appendix 2. -CROP_GROUPS: dict[str, list[int]] = { - 'corn': [1], - 'soybeans': [5], - 'wheat': [22, 23, 24], # durum, hard red spring, hard red winter -} -CROP_CODE_TO_GROUP: dict[int, str] = { - code: group for group, codes in CROP_GROUPS.items() for code in codes -} -ALL_ROW_CROP_CODES: list[int] = sorted(CROP_CODE_TO_GROUP) - -# NASS QuickStats commodity name → our crop group key. -NASS_TO_CROP_GROUP: dict[str, str] = { - 'CORN': 'corn', - 'SOYBEANS': 'soybeans', - 'WHEAT': 'wheat', -} - -# NASS QuickStats yield years averaged into the per-state yield rate. -# 4-year arithmetic mean smooths single-year outliers (methodology -# § Calculate crop yields). -NASS_YIELD_YEARS: list[int] = [2017, 2018, 2019, 2020] - -# --------------------------------------------------------------------------- -# Unit conversion constants (methodology § Unit conversion) -# --------------------------------------------------------------------------- -# Standard bushel weights per 7 CFR 810 (US Grain Standards Act), expressed -# in kg: 56 lb/bu × 0.45359237 kg/lb for corn; 60 lb/bu for soybeans and -# wheat. -BUSHEL_WEIGHT_KG: dict[str, float] = { - 'CORN': 56 * 0.45359237, - 'SOYBEANS': 60 * 0.45359237, - 'WHEAT': 60 * 0.45359237, -} -# Exact SI definition (1 acre = 0.40468564 hectares). -HA_PER_ACRE: float = 0.40468564 - -# --------------------------------------------------------------------------- -# Summary-table input asset IDs (counties and NASS yields) -# --------------------------------------------------------------------------- -GEE_TIGER_COUNTIES: str = 'TIGER/2018/Counties' -GEE_NASS_YIELDS: str = f'{GEE_ASSET_ROOT}/nass_yields_{EXPECTED_NASS_VERSION}' - -# Extend DATASET_INVENTORY with the two summary-table inputs. -DATASET_INVENTORY['tiger_counties'] = { - 'gee_asset_id': GEE_TIGER_COUNTIES, - 'resolution_hint': 'vector', - 'native_or_extracted': 'native', -} -DATASET_INVENTORY['nass_yields'] = { - 'gee_asset_id': GEE_NASS_YIELDS, - 'resolution_hint': 'state-level (tabular)', - 'native_or_extracted': 'extracted', - 'expected_version': EXPECTED_NASS_VERSION, - 'source_url': NASS_QUICKSTATS_URL_TEMPLATE.format( - date=NASS_QUICKSTATS_RELEASE_DATE - ), -} - -# --------------------------------------------------------------------------- -# Summary-table column schemas -# --------------------------------------------------------------------------- -# Single source of truth for both builders and integration tests. -# See specs/pipeline_tech_design.md § Tables. -TRANSITIONS_TABLE_COLUMNS: list[str] = [ - 'county_fips', - 'epoch_transition', - 'emissions_type', - 'total_area_ha', - 'total_emissions_tco2', - 'allocated_emissions_2020_tco2', -] -CROPS_TABLE_COLUMNS: list[str] = [ - 'county_fips', - 'crop_code', - 'crop_group', - 'total_production_kg', - 'total_production_bu', - 'total_crop_area_ha', - 'peatland_crop_area_ha', - 'yield_kg_per_ha', - 'yield_bu_per_acre', - 'total_allocated_emissions_tco2', - 'emissions_factor_kgco2e_per_kg', - 'pct_forest', - 'pct_short_veg', - 'pct_peatland_conversion', - 'pct_peatland_occupation', - 'pct_epoch_2005', - 'pct_epoch_2010', - 'pct_epoch_2015', - 'pct_epoch_2020', -] - - -def _build_crop_driver_mapping() -> dict[str, str]: - """emissions_type → pct column for the `crops` table driver breakdown. - - Groups the 9 LUC emissive types by source family: forest / wetland_forest - collapse to `pct_forest`, short_veg / wetland_short_veg to `pct_short_veg`. - Peatland rows each get their own column. `*_to_built_up` and - `forest_to_short_veg` entries exist for completeness but are filtered out - by `is_row_crop` before crop-row aggregation, since those destinations - cannot be 2020 cropland pixels. - """ - forest_family = {'forest', 'wetland_forest'} - short_veg_family = {'short_vegetation', 'wetland_short_vegetation'} - mapping: dict[str, str] = {} - for name, src, _dst in _EMISSIVE_TRANSITION_CATEGORIES: - if src in forest_family: - mapping[name] = 'pct_forest' - elif src in short_veg_family: - mapping[name] = 'pct_short_veg' - else: - raise ValueError(f'Unmapped source family for {name!r}: {src!r}') - mapping['peatland_conversion'] = 'pct_peatland_conversion' - mapping['peatland_occupation'] = 'pct_peatland_occupation' - return mapping - - -CROP_DRIVER_MAPPING: dict[str, str] = _build_crop_driver_mapping() - - -# --------------------------------------------------------------------------- -# Publish stage — BigQuery destination -# --------------------------------------------------------------------------- -# The publish stage exports the regional `transitions` and `crops` GEE -# table assets into BigQuery under compound-keyed table names of the form -# ``{PREFIX}_{region}_{transform_sha}_{publish_sha}``. Both tables land in -# the same project / dataset; the dataset location is pinned so -# ``ee.batch.Export.table.toBigQuery`` can target it explicitly. - -# ``BQ_PROJECT``, ``BQ_DATASET``, and ``BQ_JOB_LOCATION`` are defined in the -# deployment configuration block at the top of this module (US multi-region -# by default; export tasks must target the same location as the destination -# dataset). -BQ_TRANSITIONS_TABLE_PREFIX: str = 'luc_transitions' -BQ_CROPS_TABLE_PREFIX: str = 'luc_crops' - -# GCS path prefix for cloud-optimised GeoTIFF raster exports (land_use, -# emissions). Objects land at gs://{GCS_BUCKET_NAME}/{GCS_RASTER_PREFIX}/... -GCS_RASTER_PREFIX: str = f'{BQ_PROJECT:s}/{BQ_DATASET:s}/rasters' - -# GCS path prefix for CSV table exports (transitions, crops). -# Objects land at gs://{GCS_BUCKET_NAME}/{GCS_TABLE_PREFIX}/{table_name}-*.csv -GCS_TABLE_PREFIX: str = f'{BQ_PROJECT:s}/{BQ_DATASET:s}/tables' diff --git a/jdluc/utils/gee.py b/jdluc/utils/gee.py deleted file mode 100644 index 9fbe93b..0000000 --- a/jdluc/utils/gee.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Google Earth Engine client utilities. - -Thin wrappers over ``ee.data`` plus the GCS + HTTP plumbing the extract -stage needs to stage a local file into a GEE asset. Centralized here so -that per-dataset extractors in ``extract/`` stay focused on dataset- -specific logic and so the client surface is uniform across callers. - -""" - -import hashlib -import logging -import time -from typing import Any, cast - -import ee -import requests - -HIGH_VOLUME_ENDPOINT = 'https://earthengine-highvolume.googleapis.com' - -logger = logging.getLogger(__name__) - -# Poll cadence for ingestion / export task status. -POLL_INTERVAL_S: float = 30.0 -# Hard timeout for a single ingestion task — ingest should finish in -# minutes; hours means something is stuck, fail loudly. -INGESTION_TIMEOUT_S: float = 2 * 60 * 60 -# Hard timeout for a single export task. Transform-stage exports for -# CONUS land_use and emissions empirically take 130-145 min wall-clock, -# so 2 hours is too tight; this budget gives multi-hour exports headroom -# without letting truly stuck tasks linger forever. -EXPORT_TIMEOUT_S: float = 6 * 60 * 60 - - -def initialize_gee(project: str) -> None: - """Initialize Earth Engine against the high-volume endpoint. - - Args: - project: GCP project ID for Earth Engine authentication, billing, - and asset ownership. - """ - ee.Initialize(project=project, opt_url=HIGH_VOLUME_ENDPOINT) - - -def asset_exists(asset_id: str) -> bool: - """Return True iff the GEE asset exists.""" - try: - ee.data.getAsset(asset_id) - return True - except ee.EEException: - return False - - -def asset_is_populated(asset_id: str) -> bool: - """Return True iff the asset exists AND, if an ImageCollection, is non-empty. - - The extract orchestrator uses this — not bare ``asset_exists`` — to - decide whether a dataset is cache-hit. An empty ImageCollection (left - behind by ``create_image_collection_if_absent`` when a partial - ingestion failed mid-iteration) would otherwise be mistaken for a - valid cached asset and skip re-extraction, leaving downstream - transforms to mosaic 0 bands. - """ - try: - info = ee.data.getAsset(asset_id) - except ee.EEException: - return False - if info.get('type') == 'IMAGE_COLLECTION': - children = ee.data.listAssets({'parent': asset_id}).get('assets', []) - return len(children) > 0 - return True - - -def delete_asset_if_present(asset_id: str) -> None: - """Delete the asset if it exists; no-op otherwise.""" - if asset_exists(asset_id): - logger.info(f'Deleting existing asset: {asset_id}') - ee.data.deleteAsset(asset_id) - - -def create_image_collection_if_absent(asset_id: str) -> None: - """Create an empty ImageCollection asset at ``asset_id`` if absent.""" - if asset_exists(asset_id): - return - logger.info(f'Creating empty ImageCollection: {asset_id}') - # ``ee.data.createAsset``'s first argument is the asset payload; the - # stub's type is narrower than the real API accepts. - ee.data.createAsset(cast(Any, {'type': 'ImageCollection'}), asset_id) - - -def download_with_retries( - url: str, - dst_path: str, - *, - timeout_s: float = 600.0, - retries: int = 3, - backoff_s: float = 2.0, - chunk_size: int = 1 << 20, -) -> None: - """Streaming HTTP GET with exponential backoff between attempts. - - Retries transient failures (network / 5xx); raises the last exception - if all attempts fail. - """ - last_exc: Exception | None = None - for attempt in range(1, retries + 1): - try: - logger.info(f'Download attempt {attempt}/{retries}: {url} -> {dst_path}') - with requests.get(url, stream=True, timeout=timeout_s) as response: - response.raise_for_status() - with open(dst_path, 'wb') as fh: - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: - fh.write(chunk) - return - except (requests.RequestException, OSError) as exc: - last_exc = exc - logger.warning(f'Download attempt {attempt} failed: {exc}') - if attempt < retries: - time.sleep(backoff_s * attempt) - assert last_exc is not None - raise last_exc - - -def upload_to_gcs( - gcp_project: str, - bucket_name: str, - blob_name: str, - local_path: str, -) -> str: - """Stage ``local_path`` to ``gs://{bucket}/{blob}`` and return the URI.""" - # Lazy import: google.cloud.storage is a heavy transitive dep. - from google.cloud import storage - - gcs_uri = f'gs://{bucket_name}/{blob_name}' - logger.info(f'Uploading to GCS: {gcs_uri}') - client = storage.Client(project=gcp_project) - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_name) - blob.upload_from_filename(local_path) - return gcs_uri - - -def delete_gcs_blob( - gcp_project: str, - bucket_name: str, - blob_name: str, -) -> None: - """Delete a GCS staging object; logs but does not raise on NotFound.""" - from google.api_core import exceptions as gcs_exceptions - from google.cloud import storage - - logger.info(f'Deleting GCS staging object: gs://{bucket_name}/{blob_name}') - client = storage.Client(project=gcp_project) - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_name) - try: - blob.delete() - except gcs_exceptions.NotFound: - logger.warning(f'GCS blob already absent: gs://{bucket_name}/{blob_name}') - - -def start_ingestion_no_wait( - gcs_uri: str, - asset_id: str, - *, - band_name: str = 'b1', - allow_overwrite: bool = False, - request_id: str | None = None, -) -> str: - """Submit an image ingestion task and return its task ID without polling. - - Default ``request_id`` is a deterministic SHA-256 hash of ``asset_id`` - so that re-issuing the call after an orchestrator crash returns the - existing in-flight task (GEE request-id idempotency) instead of - starting a duplicate. ``force=True`` callers in the extract stage - delete the asset before submission, breaking the idempotency hash - domain (the new ingest is a different request semantically) — passing - a fresh ``request_id`` from the caller is the escape hatch if that's - ever needed. - """ - if request_id is None: - request_id = hashlib.sha256(asset_id.encode()).hexdigest()[:32] - manifest: dict[str, Any] = { - 'name': asset_id, - 'tilesets': [{'sources': [{'uris': [gcs_uri]}]}], - 'bands': [{'id': band_name}], - } - logger.info(f'Starting ingestion: {asset_id} <- {gcs_uri}') - task_info = ee.data.startIngestion( - request_id=request_id, - params=manifest, - allow_overwrite=allow_overwrite, - ) - return str(task_info['id']) - - -def wait_for_tasks( - task_ids: list[str], - *, - asset_ids_for_logging: dict[str, str] | None = None, - poll_interval_s: float = POLL_INTERVAL_S, - timeout_s: float = INGESTION_TIMEOUT_S, - heartbeat_interval_s: float | None = None, - task_labels: dict[str, str] | None = None, -) -> dict[str, str]: - """Poll task IDs in bulk until each reaches a terminal state. - - Returns ``{task_id: error_message}`` for tasks that ended FAILED or - CANCELLED; successful completions are absent from the return dict. - Raises ``TimeoutError`` if any task is still pending at ``timeout_s``. - - ``asset_ids_for_logging`` is an optional ``{task_id: asset_id}`` map - used only to make log messages identify the failed asset by name - rather than by opaque task ID. - - ``heartbeat_interval_s`` (optional) — when set, logs an INFO line at - that cadence listing the still-pending task labels and elapsed - minutes. Used by long-running export polls (transform stage); the - extract stage's tile-ingest polling leaves it None for quiet polling. - - ``task_labels`` (optional ``{task_id: label}``) — caller-friendly - kind labels (e.g. ``'transitions'``, ``'crops'``) used in heartbeat - and per-task log lines. Falls back to ``asset_ids_for_logging`` and - then the task ID when absent. - - ``getTaskStatus`` is called in chunks of 100 IDs — empirically the - largest chunk size GEE accepts without rejecting the request. At - CONUS scale (≤19 tile-based ingest tasks) the chunking is a no-op, - but it lets a global-scale (~280 task) extract stay on a single - polling loop without surfacing the chunk count to callers. - """ - pending: set[str] = set(task_ids) - failed: dict[str, str] = {} - start = time.monotonic() - deadline = start + timeout_s - next_heartbeat = start + heartbeat_interval_s if heartbeat_interval_s else None - - def _label_for(tid: str) -> str: - if task_labels and tid in task_labels: - return task_labels[tid] - if asset_ids_for_logging and tid in asset_ids_for_logging: - return asset_ids_for_logging[tid] - return tid - - while pending: - ids = list(pending) - rows: list[dict[str, Any]] = [] - for i in range(0, len(ids), 100): - rows.extend(ee.data.getTaskStatus(ids[i : i + 100])) - for row in rows: - tid = str(row.get('id', '')) - if tid not in pending: - continue - state = str(row.get('state', 'UNKNOWN')) - if state == 'COMPLETED': - # GEE reports per-task batch_eecu_usage_seconds in the - # status row; log it on completion so the run log is the - # source of truth for cost ($0.40 per EECU-hr at the list - # rate). start/update timestamps give wall time. - eecu_s = float(row.get('batch_eecu_usage_seconds', 0.0) or 0.0) - eecu_hr = eecu_s / 3600.0 - cost_usd = eecu_hr * 0.40 - start_ms = row.get('start_timestamp_ms') or 0 - end_ms = row.get('update_timestamp_ms') or 0 - wall_min = ( - (int(end_ms) - int(start_ms)) / 60000.0 - if start_ms and end_ms - else 0.0 - ) - logger.info( - f'Task {tid} for {_label_for(tid)} COMPLETED ' - f'wall={wall_min:.1f}min EECU-hr={eecu_hr:.3f} ' - f'cost_usd={cost_usd:.2f}' - ) - pending.discard(tid) - elif state in ('FAILED', 'CANCELLED'): - err = str(row.get('error_message', '')) - logger.error(f'Task {tid} for {_label_for(tid)} {state}: {err}') - failed[tid] = err - pending.discard(tid) - if pending and time.monotonic() > deadline: - raise TimeoutError( - f'wait_for_tasks: {len(pending)} task(s) pending after ' - f'{timeout_s:.0f}s' - ) - if pending and next_heartbeat is not None: - now = time.monotonic() - if now >= next_heartbeat: - elapsed_min = (now - start) / 60 - pending_labels = sorted(_label_for(tid) for tid in pending) - logger.info( - f'wait_for_tasks: still waiting on {pending_labels} ' - f'(elapsed={elapsed_min:.0f}m)' - ) - assert heartbeat_interval_s is not None - next_heartbeat = now + heartbeat_interval_s - if pending: - time.sleep(poll_interval_s) - return failed - - -def start_ingestion_and_wait( - gcs_uri: str, - asset_id: str, - *, - band_name: str = 'b1', - allow_overwrite: bool = False, - poll_interval_s: float = POLL_INTERVAL_S, - timeout_s: float = INGESTION_TIMEOUT_S, -) -> str: - """Ingest ``gcs_uri`` into ``asset_id`` and block until the task lands. - - Convenience wrapper for single-task callers (``huang_bgb``, - ``ipcc_climate_zones``); composes ``start_ingestion_no_wait`` and - ``wait_for_tasks``. Tile-based extractors (``harris_agb``, - ``gfw_peatlands``) call those primitives directly so the submit - phase can fan out across all tiles before the first poll. - - Returns the GEE task ID. Raises ``RuntimeError`` on FAILED/CANCELLED, - ``TimeoutError`` if the task hasn't completed within ``timeout_s``. - """ - task_id = start_ingestion_no_wait( - gcs_uri, - asset_id, - band_name=band_name, - allow_overwrite=allow_overwrite, - ) - failed = wait_for_tasks( - [task_id], - asset_ids_for_logging={task_id: asset_id}, - poll_interval_s=poll_interval_s, - timeout_s=timeout_s, - ) - if task_id in failed: - raise RuntimeError(f'Task {task_id} for {asset_id} failed: {failed[task_id]}') - return task_id - - -def start_table_ingestion_and_wait( - gcs_uri: str, - asset_id: str, - *, - allow_overwrite: bool = False, - poll_interval_s: float = POLL_INTERVAL_S, - timeout_s: float = INGESTION_TIMEOUT_S, -) -> str: - """Ingest ``gcs_uri`` as a FeatureCollection at ``asset_id`` and block. - - For shapefile sources, ``gcs_uri`` must point at the .zip bundle - (GEE reads .shp/.shx/.dbf/.prj out of it). For CSVs, point at the - .csv. Returns the GEE task ID. Raises on FAILED/CANCELLED or - timeout, matching ``start_ingestion_and_wait``'s error model. - """ - request_id = ee.data.newTaskId(1)[0] - manifest: dict[str, Any] = { - 'name': asset_id, - 'sources': [{'primaryPath': gcs_uri}], - } - logger.info(f'Starting table ingestion: {asset_id} <- {gcs_uri}') - task_info = ee.data.startTableIngestion( - request_id=request_id, - params=manifest, - allow_overwrite=allow_overwrite, - ) - task_id = str(task_info['id']) - failed = wait_for_tasks( - [task_id], - asset_ids_for_logging={task_id: asset_id}, - poll_interval_s=poll_interval_s, - timeout_s=timeout_s, - ) - if task_id in failed: - raise RuntimeError(f'Task {task_id} for {asset_id} failed: {failed[task_id]}') - return task_id - - -def wait_for_export_task( - task: ee.batch.Task, - asset_id: str, - *, - poll_interval_s: float = POLL_INTERVAL_S, - timeout_s: float = EXPORT_TIMEOUT_S, -) -> None: - """Block until an ``ee.batch.Task`` (Export.*.toAsset) finishes.""" - task_id = str(task.id) - failed = wait_for_tasks( - [task_id], - asset_ids_for_logging={task_id: asset_id}, - poll_interval_s=poll_interval_s, - timeout_s=timeout_s, - ) - if task_id in failed: - raise RuntimeError(f'Task {task_id} for {asset_id} failed: {failed[task_id]}') diff --git a/jdluc/utils/states.py b/jdluc/utils/states.py deleted file mode 100644 index bf399b1..0000000 --- a/jdluc/utils/states.py +++ /dev/null @@ -1,23 +0,0 @@ -"""State and region geometry helpers. - -Single-state helper (``get_multi_state_boundary``) is the transform-stage -primitive used by ``transform/land_use.py`` and ``transform/emissions.py``. -""" - -from __future__ import annotations - -import ee - -from jdluc.utils._ee_types import EEGeometry -from jdluc.utils.constants import GEE_TIGER_STATES - - -def get_multi_state_boundary(state_fips_list: list[str]) -> EEGeometry: - """Get the union of several states' boundaries as an ee.Geometry. - - Args: - state_fips_list: List of state FIPS codes (e.g., ['10', '44']). - """ - states = ee.FeatureCollection(GEE_TIGER_STATES) - filtered = states.filter(ee.Filter.inList('STATEFP', state_fips_list)) - return filtered.geometry() diff --git a/jdluc/utils/transitions.py b/jdluc/utils/transitions.py deleted file mode 100644 index ddf010f..0000000 --- a/jdluc/utils/transitions.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Land use transition encoding/decoding helpers. - -Transitions between two land use categories are packed into a single uint8: -the upper 4 bits hold from_code and the lower 4 bits hold to_code, with -0x00 reserved for the no-transition case. This scheme supports up to 16 -land use categories per side — we currently have 9. - -Two forms of each operation live in this file so the bit layout is defined -in exactly one place: scalar helpers for lookup-table construction and -.where() matchers, and raster helpers for the GEE pipeline. - -See specs/pipeline_tech_design.md § transitions.py for details. -""" - -from jdluc.utils._ee_types import EEImage - -### scalar helpers ### - - -def encode_transition(from_code: int, to_code: int) -> int: - """Pack a (from, to) category pair into a single uint8. - - Returns (from_code << 4) | to_code, or 0 when the categories are - equal (the no-transition case). - """ - assert ( - 0 <= from_code < 16 and 0 <= to_code < 16 - ), f'category codes must fit in 4 bits, got from={from_code}, to={to_code}' - if from_code == to_code: - return 0 - return (from_code << 4) | to_code - - -### raster helpers ### - - -def encode_transition_image(from_img: EEImage, to_img: EEImage) -> EEImage: - """Pack two classified category bands into a single uint8 transition band. - - Pixels where from_img == to_img are set to 0 (no-transition - sentinel); all other pixels encode (from << 4) | to. - """ - encoded = from_img.leftShift(4).bitwiseOr(to_img).uint8() - return encoded.where(from_img.eq(to_img), 0) - - -def decode_from_image(encoded: EEImage) -> EEImage: - """Extract the from category band from an encoded transition image.""" - return encoded.rightShift(4) - - -def decode_to_image(encoded: EEImage) -> EEImage: - """Extract the to category band from an encoded transition image.""" - return encoded.bitwiseAnd(0xF) diff --git a/jdluc/utils/version.py b/jdluc/utils/version.py deleted file mode 100644 index 077888f..0000000 --- a/jdluc/utils/version.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Pipeline versioning from git state. - -Exposes compute_transform_version() and compute_publish_version(), -Each version string is either {HEAD SHA[:12]} (working tree matches HEAD -for the hashed files) or {HEAD SHA[:12]}-dirty-{sha256(diff)[:8]} -(working tree differs). - -See specs/pipeline_tech_design.md § version.py for details. -""" - -import hashlib -import subprocess - -# Path prefixes (relative to repo root) whose git state feeds each stage's -# version. Both stages include utils/ because any utility change can -# affect either stage's output. -TRANSFORM_PATHS = ['jdluc/transform/', 'jdluc/utils/'] -PUBLISH_PATHS = ['jdluc/publish/', 'jdluc/utils/'] - - -def _compute_version_for_paths(paths: list[str]) -> str: - """Compute a version string derived from git state of the given paths. - - Returns the first 12 hex chars of HEAD, optionally suffixed with - -dirty-{sha256(diff)[:8]} when the working tree has staged, unstaged, - or untracked changes under any of paths. - """ - repo_root = subprocess.check_output( - ['git', 'rev-parse', '--show-toplevel'], text=True - ).strip() - head_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() - - # Staged + unstaged changes limited to the requested path prefixes. - tracked_diff = subprocess.check_output( - ['git', 'diff', 'HEAD', '--'] + paths, - cwd=repo_root, - text=True, - ) - - # Untracked files under any of the requested path prefixes. - untracked_files = subprocess.check_output( - ['git', 'ls-files', '--others', '--exclude-standard', '--'] + paths, - cwd=repo_root, - text=True, - ).strip() - - untracked_content = '' - if untracked_files: - for fpath in untracked_files.splitlines(): - try: - with open(f'{repo_root}/{fpath}') as f: - untracked_content += f'--- untracked: {fpath}\n{f.read()}\n' - except OSError: - pass - - combined_diff = tracked_diff + untracked_content - - if not combined_diff.strip(): - return head_sha[:12] - - diff_hash = hashlib.sha256(combined_diff.encode()).hexdigest()[:8] - return f'{head_sha[:12]}-dirty-{diff_hash}' - - -def compute_transform_version() -> str: - """Version string for transform-stage assets: hashes transform/ + utils/.""" - return _compute_version_for_paths(TRANSFORM_PATHS) - - -def compute_publish_version() -> str: - """Version string for publish-stage assets: hashes publish/ + utils/.""" - return _compute_version_for_paths(PUBLISH_PATHS) diff --git a/pyproject.toml b/pyproject.toml index f27f933..5ca85c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,17 +6,21 @@ readme = "README.md" requires-python = ">=3.14,<3.15" dependencies = [ - "earthengine-api", - "geemap", - "google-api-python-client", - "google-auth-oauthlib", - "google-cloud-bigquery", + "GitPython", + "dask", + "gcsfs", + "geopandas", "google-cloud-storage", "netCDF4", + "networkx", "pandas", + "pyarrow", + "python-dotenv", "requests", + "rio-cogeo", "rioxarray", "xarray", + "zarr", ] [dependency-groups] @@ -25,6 +29,8 @@ dev = [ "pandas-stubs", "pre-commit", "pytest", + "types-geopandas", + "types-networkx", "types-requests", ] @@ -35,27 +41,13 @@ build-backend = "hatchling.build" [tool.pytest.ini_options] addopts = "-s -v -m 'not integration'" markers = [ - "integration: Integration tests that require GEE / GCS / BQ credentials.", + "integration: tests requiring live GCS credentials; excluded by default (run with `-m integration`).", ] filterwarnings = [ "ignore::DeprecationWarning", ] log_cli_level = "INFO" -[tool.ruff] -preview = true -line-length = 88 -lint.extend-ignore = [ - "BLE001", # blind-except — used intentionally in extract / publish wrappers - "SIM117", # nested with statements — clearer when patches are listed - "SIM103", # collapsing two-line is/None branch reads worse here - "UP007", # `Union[X, Y]` → `X | Y` (we still use Union in some signatures) - "RUF015", # `next(iter(...))` over `[0]` slice — both are fine -] - -[tool.ruff.lint.per-file-ignores] -"**/__tests__/**/*.py" = ["ARG001", "ARG002"] - [[tool.mypy.overrides]] module = [ "google", @@ -63,6 +55,8 @@ module = [ "google.cloud", "google.cloud.storage", "google.cloud.storage.*", + "rasterio", + "rasterio.*", ] ignore_missing_imports = true follow_imports = "skip" @@ -72,11 +66,12 @@ implicit_reexport = true root_package = "jdluc" [[tool.importlinter.contracts]] -name = "ETL pipeline direction" +name = "ETL" type = "layers" layers = [ - "jdluc.pipeline", - "jdluc.publish", - "jdluc.transform", - "jdluc.extract", + "jdluc.emissions_factors", + "jdluc.attribution", + "jdluc.emissions", + "jdluc.harmonize", + "jdluc.ingest", ] diff --git a/specs/methodology.md b/specs/methodology.md index 846cb67..af70bd7 100644 --- a/specs/methodology.md +++ b/specs/methodology.md @@ -7,7 +7,7 @@ This document describes an LSRS-compliant jdLUC methodology for estimating land The core of the methodology is the Global Land Cover and Land Use Change dataset produced by the University of Maryland and the Land & Carbon Lab (GLAD GLC; CC BY 4.0). This dataset provides the main underlying land use time series. It provides land cover at 5-year epochs from 2000-2020, at 30m resolution. We chose to build off this dataset because: - **Accuracy**: An [independent validation](https://landcarbonlab.org/insights/global-land-cover-maps-accuracy-applications/) by Land & Carbon Lab and Wageningen University found GLAD GLC had the second-highest global accuracy of the leading high resolution land cover datasets, with the best forest accuracy. The dataset with the highest overall accuracy (ESA's WorldCover) has only 2020 and 2021 maps, generated with different algorithms, making it unsuitable for change detection. -- **Temporal consistency**: GLC uses the same classification model across all epochs (2000, 2005, 2010, 2015, 2020), which ensures detected changes reflect real land cover change rather than methodological drift. +- **Temporal consistency**: GLC uses the same classification model across all epochs (2000, 2005, 2010, 2015, 2020), which ensures detected changes reflect real land cover change rather than methodological drift. - **Simplicity**: GLC covers all land cover types, which eliminates the need to reconcile different datasets for forests, grasslands, etc. - **Historical coverage**: GLC's coverage extends back to 2000, which eliminates the need for backfill methodologies. - **Global extensibility**: GLC works worldwide. @@ -22,35 +22,37 @@ Finally, we use the USDA Cropland Data Layer to identify the specific row crop g ### Pixels -We use the GLAD GLC native 30-meter resolution (Landsat-based, 0.00025 degrees) as the common geospatial reference for all data. All other raster inputs are resampled and/or reprojected to align with this grid. +We use the GLAD GLC native 30-meter resolution (Landsat-based, 0.00025 degrees) as the common geospatial reference for all data. All other raster inputs are resampled and/or reprojected to align with this grid. Source datasets are ingested as 10° tiles and mosaicked onto this common grid (via GDAL VRT warping) at 4,000 pixels per degree. -### State boundaries +### Jurisdiction boundaries -We use the US Census Bureau TIGER/Line state boundaries to assign each pixel to a US state. The data is available in the GEE catalog as `TIGER/2018/States`. For pixels that overlap state boundaries, we assign to the state containing the pixel centroid. +We assign each pixel to a jurisdiction using the World Bank Official Boundaries (CC BY 4.0), a three-level administrative hierarchy: Admin 0 (national), Admin 1 (provincial/state), and Admin 2 (district/county). US states correspond to the provincial (Admin 1) level, with World Bank admin IDs `USA001`…`USA051`; the `STATE_FIPS_TO_ADMIN_ID` bridge in `jdluc/datasets/usda_nass_quickstats.py` maps US Census state FIPS codes to these IDs. Pixels are assigned by clipping the emissions raster to each jurisdiction polygon (`rio.clip`, `drop=False`), not by pixel centroid. Using a global administrative hierarchy in place of US-only state boundaries lets the same pipeline roll emissions up to any jurisdiction worldwide. + +*Reference:* World Bank Official Boundaries, https://datacatalog.worldbank.org/search/dataset/0038272/world-bank-official-boundaries (CC BY 4.0). ## Cataloging emissions drivers -For every ~30m pixel in the continental United States, we build a time series of land-cover transitions between the five epochs in the GLAD GLC dataset: 2000 → 2005 → 2010 → 2015 → 2020. +For every ~30m pixel in the continental United States, we build a time series of land-cover transitions between the five epochs in the GLAD GLC dataset: 2000 → 2005 → 2010 → 2015 → 2020. In addition, we catalogue a binary value specifying whether the pixel is peatland and each pixel's IPCC climate domain. ### Land cover transitions -We load the GLAD GLCLUC v2 combined maps from GEE at `projects/glad/GLCLU2020/v2/LCLUC_{year}`. +We ingest the GLAD GLCLUC v2 combined annual maps directly from the published Hansen GeoTIFFs at `https://storage.googleapis.com/earthenginepartners-hansen/GLCLU2000-2020/v2/{year}/{tile_id}.tif`. -GLAD GLCLUC encodes land cover as unsigned 8-bit values. We simplify to land use categories as follows: +GLAD GLCLUC encodes land cover as unsigned 8-bit values. We collapse them into the 7-member `LandClass` enum (`jdluc/datasets/glad_glcluc.py`): -| GLAD GLCLUC pixel values | Land use category | +| Land class | GLAD GLCLUC values | |---|---| -| 25–48 | Forest (terra firma tree cover, by canopy height 3m to >25m) | -| 125–148 | Wetland forest | -| 1–24 | Short vegetation (grassland, shrubland, sparse vegetation gradient) | -| 100–124 | Wetland short vegetation | -| 244 | Cropland | -| 250 | Built-up | -| 200–207 | Water | -| 241 | Snow/ice | -| 0 | Bare (3% vegetation cover or less) | +| Forest | 25–48 and 125–148 (terra-firma + wetland forest) | +| Grassland | 0–24 and 100–124 (short vegetation + wetland short vegetation, including bare ground) | +| Cropland | 244 | +| Built-up | 250 | +| Water | 200–207 | +| Snow/ice | 241 | +| Ocean | 254 | + +This collapses the nine categories used in earlier iterations. Terra-firma and wetland forest are merged into a single **Forest** class, and wetland short vegetation is merged into **Grassland**. Bare ground (value 0) is no longer a standalone category: it is folded into Grassland (per the code comment, "although the POC considers 0 as bareground, we include it as grassland here"). As a result, bare pixels carry grassland vegetation carbon and can act as emissive sources when converted. Finally, **Ocean** (254) is split out as a distinct class from inland **Water** (200–207). We compare consecutive GLAD GLCLUC epochs (2000→2005, 2005→2010, 2010→2015, 2015→2020), and for each pixel, record whether a transition occurred, from which land use category, and to which. We record all transitions, but the only transitions that generate LUC emissions are those where land cover changes from a higher-carbon state (e.g. forest, short vegetation/grassland) to a lower-carbon state (e.g. cropland, built-up). Transitions in the reverse direction (e.g., cropland → forest) would represent carbon removals, which are not currently included in this methodology (see Appendix 2). @@ -73,7 +75,7 @@ All input layers are rasterized or resampled to the 30m Hansen Global Forest Cha ### Climate domain identification -We use an independently recreated raster (Lewis, 2022) to determine the IPCC climate domain for each pixel. This dataset is built following the IPCC 2019 Refinement decision tree (Vol 4, Ch 3, Annex 3A.5), which classifies climate zones using mean annual temperature, precipitation, potential evapotranspiration ratio, frost days, and elevation. The raster is provided at 0.5° resolution (~50km), which is sufficient for climate domain classification. We use 10 of the 12 IPCC climate zones, excluding polar zones (negligible cropland). +We use an independently recreated raster (Lewis, 2022) to determine the IPCC climate domain for each pixel. This dataset is built following the IPCC 2019 Refinement decision tree (Vol 4, Ch 3, Annex 3A.5), which classifies climate zones using mean annual temperature, precipitation, potential evapotranspiration ratio, frost days, and elevation. The raster is provided at 0.5° resolution (~50km), which is sufficient for climate domain classification. We use 10 of the 12 IPCC climate zones, excluding polar zones (negligible cropland). We store the climate domain as a single categorical value per pixel (not a time series). @@ -87,14 +89,14 @@ For each pixel with a detected land use transition, emissions are the sum of car 1. Vegetation carbon, defined as: 1. Above-ground biomass - 2. Root biomass + 2. Root biomass 3. Dead organic matter (forests only) 2. Soil organic carbon 1. Mineral soil stock change 2. Peatland drainage -For forests, we calculate 1.a-1.c separately. For grasslands and shrublands, we use a single overall value for vegetation. In both cases, soil organic carbon is a separate calculation, and is handled differently for mineral soils and peatland pixels. +For forests, we calculate 1.a-1.c separately. For grasslands and shrublands, we use a single overall value for vegetation. In both cases, soil organic carbon is a separate calculation, and is handled differently for mineral soils and peatland pixels. All categories of emissions are initially calculated as if the transition is instantaneous. Emissions are then subsequently allocated over time using the linear discounting approach described in the next section, following GHGP guidance. @@ -106,10 +108,10 @@ Finally, we add in an additional ongoing (land management) component of peatland When a forest is converted to another land cover type, we calculate: -- for forest → grassland conversions, loss of the difference in vegetation carbon between the forest and the grassland +- for forest → grassland conversions, loss of the difference in vegetation carbon between the forest and the grassland - for forest → cropland or built-up land cover conversions, full loss of forest vegetation carbon -Forest vegetation carbon is calculated as follows. Soil carbon losses from these transitions are handled separately in the next section. +Forest vegetation carbon is calculated as follows. ##### Above ground live biomass @@ -125,7 +127,7 @@ Below-ground biomass for forests comes from Huang et al. (2021) at ~1km resoluti ##### Dead organic matter -Dead organic matter (dead wood + litter) is estimated as a fraction of above-ground biomass, following the UNFCCC CDM AR-TOOL-12 methodology. +Dead organic matter (dead wood + litter) is estimated as a fraction of above-ground biomass, following the UNFCCC CDM AR-TOOL-12 methodology. ``` DOM_C = above_ground_biomass × (dead_wood_factor × 0.50 + litter_factor × 0.37) @@ -142,7 +144,7 @@ Where AGB is calculated as described in the previous section, 0.50 (dead wood, t | Tropical, >2000m (montane) | 0.07 | 0.01 | | Temperate / Boreal | 0.08 | 0.04 | -For CONUS, nearly all forest pixels fall in the temperate/boreal row (dead wood = 8% of AGB, litter = 4%). For a typical US temperate forest with AGB = 150 Mg/ha, this yields DOM carbon of ~8.2 tC/ha (dead wood 6.0 + litter 2.2). +For CONUS, nearly all forest pixels fall in the temperate/boreal row (dead wood = 8% of AGB, litter = 4%). For a typical US temperate forest with AGB = 150 Mg/ha, this yields DOM carbon of ~8.2 tC/ha (dead wood 6.0 + litter 2.2). *References:* - CDM AR-TOOL-12 v3.0: Estimation of carbon stocks and change in carbon stocks in dead wood and litter in A/R CDM project activities. UNFCCC. @@ -153,18 +155,18 @@ For CONUS, nearly all forest pixels fall in the temperate/boreal row (dead wood For grassland and shrubland, we assign a single total vegetation carbon density, which represents above ground biomass + below ground biomass. Values are adapted from the BLUE model (Hansis et al. 2015, supplementary Table S1), used in IPCC assessments and the Global Carbon Budget. We remap BLUE's PFT-level carbon densities to IPCC climate domains; the assignments for tropical dry, tropical montane, warm/cool temperate dry, and boreal moist climate zones represent our interpretation where the source provides no direct equivalent. -| IPCC climate domain | Total vegetation C (tC/ha) | -| ------------------------ | -------------------------- | -| Tropical, Wet | 18 | -| Tropical, Moist | 18 | -| Tropical, Dry | 7 | -| Tropical, Montane | 7 | -| Warm Temperate, Moist | 7 | +| IPCC climate domain | Total vegetation C (tC/ha) | +| ------------------------ | -------------------------- | +| Tropical, Wet | 18 | +| Tropical, Moist | 18 | +| Tropical, Dry | 7 | +| Tropical, Montane | 7 | +| Warm Temperate, Moist | 7 | | Warm Temperate, Dry | 5 | -| Cool Temperate, Moist | 7 | +| Cool Temperate, Moist | 7 | | Cool Temperate, Dry | 5 | -| Boreal, Moist | 6 | -| Boreal, Dry | 3 | +| Boreal, Moist | 6 | +| Boreal, Dry | 3 | For the majority of CONUS short vegetation conversions (Great Plains grassland in the warm/cool temperate zones), these values are at the high end of total vegetation carbon estimated from field-measured above-ground biomass and standard root-to-shoot ratios (Mokany et al., 2006; IPCC 2006 Vol 4, Table 6.1), providing slight conservatism for short/mixed grass prairie. However they underestimate vegetation carbon slightly for tallgrass prairie and significantly for western shrublands like California chaparral. Literature values for comparison: @@ -203,7 +205,7 @@ _References:_ We calculate soil organic carbon (SOC) losses for non-peatland pixels using the IPCC stock change approach (IPCC 2019, Vol 4, Ch 5, Tables 5.5 and 5.10). This follows the IPCC approach of treating mineral and organic soils separately. See IPCC 2006 Vol 4, Ch 5, Section 5.2.3.2. -SoilGrids SOC stock is available in GEE at `projects/soilgrids-isric/ocs_mean` (0–30cm stock, units t/ha). Where SoilGrids has NoData (masked pixels), we use the IPCC default reference SOC stock for warm temperate moist mineral soil (low activity clay): 63 tC/ha (IPCC 2019 Refinement, Vol 4, Table 2.3). This affects <1% of conversion pixels. +SoilGrids 0–30cm SOC stock (units t/ha) is ingested as a source dataset and resampled to the common grid. Masked / NoData SoilGrids pixels are handled by the pipeline's standard no-data unification. For each pixel with a land use transition, we compute the SOC change as `SOC_stock × (1 - F_LU)`, where `F_LU` is the IPCC Tier 1 land-use stock change factor for the destination land use under the pixel's climate domain. We convert to tCO2 by multiplying by pixel area in hectares and the CO2/C mol ratio (44/12). @@ -223,10 +225,10 @@ We set input and management factors to 1 (nominal input, full tillage) per IPCC Peatland pixels are the most complex and novel component of the methodology. We use a two-part model that captures (1) a declining emission pulse over the first 20 years post-drainage, and (2) a flat steady-state emission rate thereafter. The initial pulse is modeled as land use change under the GHGP framework, with GHGP's standard linear discounting providing the decline curve. The flat steady state emissions are modeled as annual land management emissions. Both values are calibrated based on the IPCC 2013 Wetlands Supplement Tier 1 emission factors and the scientific literature on rate of peatland emissions decay: -- Land use change = 621 t CO₂ ha⁻¹ -- Land management = 37.3 t CO₂-eq ha⁻¹ yr⁻¹ +- Land use change = 621 t CO₂ ha⁻¹ +- Land management = 37.3 t CO₂-eq ha⁻¹ yr⁻¹ -The detailed model structure and derivation of these values is described in the [supplement](peatland_methodology_supplement.md). +The detailed model structure and derivation of these values is described in the [supplement](peatland_methodology_supplement.md). ## Allocating emissions to crop years @@ -249,7 +251,7 @@ The GHGP-prescribed linear discounting weights are calculated as: weight(year) = 10.25% - 0.5% × years_since_conversion ``` -Where `years_since_conversion = reporting_year - conversion_year + 1 (ranging from 1 to 20)`. Since our transition data only exists at GLAD epoch boundaries, we aggregate these 20 per-year weights into 4 epoch weights by averaging across the conversion years each epoch covers. +Where `years_since_conversion = reporting_year - conversion_year + 1 (ranging from 1 to 20)`. Since our transition data only exists at GLAD epoch boundaries, we aggregate these 20 per-year weights into 4 epoch weights, one per transition. Each GLAD GLC map represents land cover at approximately the labeled year. A transition detected between consecutive maps therefore is predicted by the model to have occurred in one of the 5 years following the earlier map. We treat those 5 candidate conversion years as equally likely and use the unbiased estimator: the arithmetic mean of the 5 corresponding GHGP per-year weights. For the 2020 reporting year this gives: @@ -260,7 +262,7 @@ Each GLAD GLC map represents land cover at approximately the labeled year. A tra | 2010→2015 | 2011–2015 | 8 | 6.25% | | 2015→2020 | 2016–2020 | 3 | 8.75% | -Note that these weights are *not* the GHGP weight evaluated at the continuous midpoint of each interval (e.g., 2017.5 for 2015→2020, which would give 8.50% rather than 8.75%). Because the GHGP weight schedule is defined on discrete integer years rather than as a continuous function of time, the unbiased per-year average is offset by 0.5 years from the continuous midpoint. For any individual pixel, the maximum residual timing error is ±2 years, corresponding to ±1 percentage point on the weight — negligible relative to the overall uncertainty of the model. +Note that these weights are *not* the GHGP weight evaluated at the continuous midpoint of each interval (e.g., 2017.5 for 2015→2020, which would give 8.50% rather than 8.75%): because the GHGP schedule is defined on discrete integer years, the unbiased per-year average is offset by 0.5 years from the continuous midpoint. The difference is negligible relative to the overall uncertainty of the model. ## Allocating emissions to crops @@ -271,13 +273,13 @@ For each 30m pixel classified as cropland by GLAD GLC 2020 and as a row crop by 3. Apply the state-level average yield rate (kg/ha) 4. Multiply by pixel area in hectares -The CDL crop layer for 2020 is available on GEE at `USDA/NASS/CDL/2020`. +CDL is ingested as a source dataset (`jdluc/datasets/usda_nass_cdl.py`); crop codes are resolved via the `CropClass` enum (corn = 1, soybeans = 5, wheat = 22/23/24). The GLAD-cropland filter — restricting CDL pixels to those where `land-class:2020 == CROPLAND` — is an explicit, toggleable step (`attribution.skip_glad_crop_filter`). Because we apply crop yields only for those CDL pixels also identified as cropland by GLAD GLC 2020, we lose ~9% of total US production; however, since the excluded pixels are absent from both the emissions numerator and the production denominator, they should not significantly affect our final emissions factors on average. See Appendix 1 for details. ### Calculate crop yields -We use state-level USDA yield statistics to estimate 2020 crop production per pixel. We use the USDA National Agricultural Statistics Service (NASS) QuickStats crops dataset, available at https://www.nass.usda.gov/datasets/, filtering for: +We use state-level USDA yield statistics to estimate 2020 crop production per pixel. Yields are pulled live from the USDA National Agricultural Statistics Service (NASS) QuickStats API and ingested as a `TabularDataset` (`jdluc/datasets/usda_nass_quickstats.py`, requires an API key), filtering for: - `STATISTICCAT_DESC = 'YIELD'` - `AGG_LEVEL_DESC = 'STATE'` @@ -307,7 +309,7 @@ Where `HA_PER_ACRE = 0.40468564`. For each (state, crop) combination, we compute the average emissions factor as: ``` -EF[state, crop] = Σ(allocated_emissions[pixel]) / Σ(yield_kg_per_ha × pixel_area_ha) +EF[state, crop] = Σ(allocated_emissions[pixel]) / Σ(yield_kg_per_ha × pixel_area_ha) ``` Where the sum runs over all 30m pixels classified as that crop by CDL 2020 and as cropland by GLAD GLC 2020 within the state. The numerator uses the 2020-allocated emissions from the previous section (linearly discounted LUC plus peatland land management). The denominator is total state production for that crop calculated as the per-ha yield x pixel area for the same pixel set. @@ -315,14 +317,14 @@ National emissions factors are computed analogously by summing across CONUS. ## Final outputs -The pipeline produces two tables. Full schemas live in `pipeline_tech_design.md`. +The pipeline produces two main artifacts: -- **`transitions`** — one row per (county, epoch_transition, emissions_type). Gives the area converted under each emissive transition (forest→cropland, short-veg→built-up, etc.), the gross emissions from that transition, and the 20-year linearly discounted allocation to 2020. Rolls up to per-state and CONUS totals, and to per-source-family or per-epoch summaries. -- **`crops`** — one row per (county, crop_group). Gives the per-crop emissions factor in kgCO2e per kg of crop, 2020 production and area, and the breakdown of allocated emissions by source family (forest, short-veg, peatland conversion, peatland occupation) and epoch. +- **Per-pixel emissions (zarr)** — written by `emissions.workflow`. Variables include the per-year land classes (`land-class:{year}`); the carbon-stock layers `aboveground-carbon`, `belowground-carbon`, `dead-organic-matter-carbon`, and `grassland-carbon` (all in `:tcarbon-per-ha`); `vegetation-carbon:{year}`; the per-epoch fluxes `vegetation-emissions:{b}-{a}`, `soil-emissions:{b}-{a}`, and `emissions:{b}-{a}`; `peatland-occupation`; `emissions-per-hectare`; `hectares-per-pixel`; and total `emissions`. Variable names carry unit suffixes (e.g. `emissions-per-hectare:tco2e-per-ha`). +- **Per-(admin level, crop, jurisdiction) emissions-factor table** — produced by `emissions_factors.workflow`. Columns: `crop_hectares`, `peatland_crop_hectares`, `peatland_occupation_emissions`, `total_emissions`, `total_production_kg`, `emissions_factor_kgco2e_per_kg`, and `peatland_occupation_fraction`. ## Appendix 1: GLAD GLC vs CDL row crop comparison -As described above, the methodology calculates emissions for pixels identified by GLC as cropland, using CDL to allocate among crops. This means that any CDL pixels not identified as cropland by GLAD GLC 2020 are excluded from both emissions and production. +As described above, the methodology calculates emissions for pixels identified by GLC as cropland, using CDL to allocate among crops. This means that any CDL pixels not identified as cropland by GLAD GLC 2020 are excluded from both emissions and production. To test whether this exclusion likely biases our EFs, we computed confusion matrices crossing CDL row crop classification against GLAD GLC cropland (pixel value 244) for all 48 CONUS states + DC, comparing CDL 2020 × GLAD GLC 2020. @@ -332,7 +334,7 @@ To test whether this exclusion likely biases our EFs, we computed confusion matr | **CDL not row crop** | 37,238,049 ha | 631,957,357 ha | 669,195,406 ha | | **Total** | 136,713,915 ha | 642,205,412 ha | 778,919,327 ha | -90.7% of CDL row crop pixels are also identified as crops by GLAD. +90.7% of CDL row crop pixels are also identified as crops by GLAD. The 9.3% of "lost" CDL row crops are concentrated in regions with smaller, more fragmented fields: @@ -365,7 +367,7 @@ These pixels fall into two cases, which we analyzed for 10 key agricultural stat | Wetland | 66,092 | 1% | Would generate forest or grassland emissions. | | Water/bare/other | 8,429 | <1% | Negligible | -Because lost pixels are excluded from both the emissions numerator and the production denominator, what matters for EF accuracy is whether they are systematically different from the in-scope population. If so, it appears that it's in a way that biases the EF slightly upward rather than downward: only ~3% of lost pixels have a GLC history showing conversion from a non-crop source, compared with the ~6.5% conversion-from-non-crop rate observed among the in-scope CDL row crop population. The dropped pixels are, if anything, biased toward stable, non-converting land. +Because lost pixels are excluded from both the emissions numerator and the production denominator, what matters for EF accuracy is whether they are systematically different from the in-scope population. If so, it appears that it's in a way that biases the EF slightly upward rather than downward: only ~3% of lost pixels have a GLC history showing conversion from a non-crop source, compared with the ~6.5% conversion-from-non-crop rate observed among the in-scope CDL row crop population. The dropped pixels are, if anything, biased toward stable, non-converting land. See `analyses/cdl_glad_glc_comparison.ipynb` for the full analysis. @@ -375,7 +377,7 @@ This methodology is a first draft. We see a number of areas where further resear ### GLAD GLC vs Hansen TCL forest-detection globally -Although forest conversion emissions are very small in the US, they will become the dominant source of emissions as we expand globally. Our preliminary analysis (offline, not yet published in this repo) shows significant differences in the US in forest detections between the GLAD GLC layer we are using and the widely used Global Forest Watch tree cover loss dataset designed specifically for this purpose. There are few enough conversion events in the US that these disagreements could be noise. But knowing we want to extend globally over time, and with our land cover layer as the single most important methodology choice, we should do more investigation of this issue early. +Although forest conversion emissions are very small in the US, they will become the dominant source of emissions as we expand globally. Our preliminary analysis (offline, not yet published in this repo) shows significant differences in the US in forest detections between the GLAD GLC layer we are using and the widely used Global Forest Watch tree cover loss dataset designed specifically for this purpose. There are few enough conversion events in the US that these disagreements could be noise. But knowing we want to extend globally over time, and with our land cover layer as the single most important methodology choice, we should do more investigation of this issue early. **Potential impact:** Minimal for US row-crop EFs; potentially substantial in other geographies. @@ -396,7 +398,7 @@ Although forest conversion emissions are very small in the US, they will become ### Peatland dataset choice -**Issue**: We use the GFW Global Peatlands raster composite. For CONUS, GFW uses Xu PEATMAP above 40°N; below 40°N it falls back to Gumbricht et al. (2017), a tropical-tuned hydrological model. This cutoff may be in the wrong spot for the temperate US — it affects Delaware, the mid-Atlantic, the Southeast, southern California, Arizona, New Mexico, and most of Texas. +**Issue**: We use the GFW Global Peatlands raster composite. For CONUS, GFW uses Xu PEATMAP above 40°N; below 40°N it falls back to Gumbricht et al. (2017), a tropical-tuned hydrological model. This cutoff may be in the wrong spot for the temperate US — it affects Delaware, the mid-Atlantic, the Southeast, southern California, Arizona, New Mexico, and most of Texas. **Potential impact:** National-level impact is likely modest because the Corn Belt sits above 40°N where Xu is the active source anyway; the issue concentrates in Southeast and mid-Atlantic states. @@ -406,7 +408,7 @@ Although forest conversion emissions are very small in the US, they will become **Issue:** GLAD GLC's "short vegetation" category (values 1–24) encodes only vegetation cover fraction (~7% to 100% cover), not vegetation type (Potapov et al., 2022). It does not distinguish grassland from shrubland. In the face of this limitation, the current methodology uses simple climate-zone-stratified carbon stock from the Houghton/BLUE bookkeeping parameterization for these pixels. The Houghton/BLUE values seem reasonably well-calibrated for herbaceous grassland (the dominant short vegetation type in US cropland conversion areas), but undercount AGB for woody shrubland: sagebrush steppe has 3–4 tC/ha (Fusco et al., 2019), and mature California chaparral has 17–28 tC/ha (Bohlman et al., 2018). Because most US short vegetation → cropland conversion occurs on Great Plains grassland rather than shrubland, the impact on national-level emission factors is likely small, but the undercount could be material for state-level factors in shrubland-heavy states, and when the methodology is extended globally. -**Potential impact:** Grassland conversion is the dominant driver of emissions for row crops in the United States. Even if shrubland is only 10-20% of the "short vegetation" conversions, the underestimate on those pixels could be large enough to matter. +**Potential impact:** Grassland conversion is the dominant driver of emissions for row crops in the United States. Even if shrubland is only 10-20% of the "short vegetation" conversions, the underestimate on those pixels could be large enough to matter. **Potential improvement path**: Two approaches, in increasing order of sophistication: (1) Overlay an auxiliary classification that distinguishes shrubland from grassland (e.g., ESA WorldCover at 10m or NLCD Shrub/Scrub class) and apply differentiated literature-based carbon densities (~6–12 tC/ha for sagebrush, ~17–28 tC/ha for chaparral, vs. the current 5–7 tC/ha for all short vegetation). (2) Replace the static lookup table entirely with satellite-derived, spatially explicit AGB estimates for non-forest vegetation, using a product like IB-AGC (Li et al., 2025) at 25km resolution, subtracting known forest and crop biomass contributions, and distributing the residual across 30m grassland/shrubland pixels as a continuous function of woody fractional cover from the Copernicus Global Land Service. @@ -418,7 +420,7 @@ Although forest conversion emissions are very small in the US, they will become ### Missing carbon sequestration -**Issue**: The methodology tracks only emissions from land use change, not carbon removals when cropland reverts to forest or grassland. This is consistent with the GHGP LSRS's approach, but it would be helpful to have carbon sequestration values available for comparing to national inventories or potentially for use of the dataset in LMU-level analyses. +**Issue**: The methodology tracks only emissions from land use change, not carbon removals when cropland reverts to forest or grassland. This is consistent with the GHGP LSRS's approach, but it would be helpful to have carbon sequestration values available for comparing to national inventories or potentially for use of the dataset in LMU-level analyses. **Potential impact:** The US has had significant cropland→forest reversion (e.g. CRP enrollment, eastern reforestation). This is likely significant in any circumstance where GHGP allows these emissions to be counted. @@ -426,19 +428,19 @@ Although forest conversion emissions are very small in the US, they will become **Issue**: The CDM AR-TOOL-12 DOM factors for tropical forests appear to be an underestimate relative to US FIA field measurements, which show a national average of ~20 tC/ha total DOM (~10 tC/ha dead wood + ~10 tC/ha litter; Domke et al., 2016; Woodall et al., 2008) -- ~2.4x the 8.2 tC/ha typical value we calculated above. We've chosen to stick with the well-standardized and peer reviewed CDM AR-TOOL-12 approach for now, but it would be good to investigate and understand this difference. One early hypothesis is a definitional difference: FIA "forest floor" may include duff/humus (partially decomposed organic material above mineral soil), that is classified as soil rather than litter in the CDM/IPCC framework. Another factor could be that the 8.2 tC/ha "typical" value quoted above is below the area-weighted US average, which could be pulled up by outliers. -**Potential impact:** The ~2.4x gap between CDM AR-TOOL-12 (~8.2 tC/ha) and FIA measurements (~20 tC/ha) translates to ~43 tCO2/ha missing per forest pixel. That's roughly 8-12% of typical per-pixel forest conversion emissions. If forest conversion emissions are only 10% of total row crop emissions, that's ~1% of total emissions. But the gap may be partly definitional (FIA "forest floor" includes duff/humus classified as soil under IPCC), so the real impact could be considerably smaller. Worth investigating but uncertain. +**Potential impact:** The ~2.4x gap between CDM AR-TOOL-12 (~8.2 tC/ha) and FIA measurements (~20 tC/ha) translates to ~43 tCO2/ha missing per forest pixel. That's roughly 8-12% of typical per-pixel forest conversion emissions. If forest conversion emissions are only 10% of total row crop emissions, that's ~1% of total emissions. But the gap may be partly definitional (see above), so the real impact could be considerably smaller. Worth investigating but uncertain. ### Harris above ground forest biomass: static year-2000 values -**Issue**: Harris et al. (2021) provides circa year-2000 biomass. For forest loss events in later years, actual biomass may differ due to growth or partial disturbance. +**Issue**: Harris et al. (2021) provides circa year-2000 biomass. For forest loss events in later years, actual biomass may differ due to growth or partial disturbance. -**Potential impact:** Although grassland conversion is the dominant driver of U.S. row crop-driven emissions, forest conversion dominates per-hectare emissions where conversion events do occur. And although for US temperate forests 20 years of growth is a relatively small fraction of total standing biomass, the most recent transitions (2015-2020) get the highest allocation weights (8.75%) while also potentially having the largest AGB underestimate (potentially 15-30% growth over 15-20 years). Thus we could be underestimating total forest conversion emissions by 10-20% for recently cleared areas. +**Potential impact:** Although grassland conversion is the dominant driver of U.S. row crop-driven emissions, forest conversion dominates per-hectare emissions where conversion events do occur. And although for US temperate forests 20 years of growth is a relatively small fraction of total standing biomass, the most recent transitions (2015-2020) get the highest allocation weights (8.75%) while also potentially having the largest AGB underestimate (potentially 15-30% growth over 15-20 years). Thus we could be underestimating total forest conversion emissions by 10-20% for recently cleared areas. ### GLAD GLC forest definition vs. Accountability Framework 10% canopy threshold **Issue**: The Accountability Framework / SBTi FLAG guidance provides specific rules on what degree of forest cover should be treated as forest. GLAD GLC defines forest by canopy height (values 25–48 = 3m to >25m trees). These definitions may not be perfectly equivalent. The task is to verify alignment and assess any differences in forest extent. Combined with the issues above, this might also push us to take a less categorical approach to estimation of above ground carbon stocks. The physical reality is that forest -> shrubland -> grassland is more a continuum than a set of discrete categories. -**Potential impact:** This shifts pixels between the forest category (high per-hectare emissions) and the short vegetation category (lower per-hectare emissions). The magnitude depends on how much area lies at the boundary between the two definitions. In US temperate forests, the 3m canopy height threshold probably captures most of what a 10% canopy cover threshold would — the ambiguous zone is likely sparse woodland and savanna edges. Maybe a few percent impact on total emissions, concentrated in transition zones. +**Potential impact:** This shifts pixels between the forest category (high per-hectare emissions) and the short vegetation category (lower per-hectare emissions). The magnitude depends on how much area lies at the boundary between the two definitions. In US temperate forests, the 3m canopy height threshold probably captures most of what a 10% canopy cover threshold would — the ambiguous zone is likely sparse woodland and savanna edges. Maybe a few percent impact on total emissions, concentrated in transition zones. ### GLAD GLC built-up classification is over-inclusive vs. NLCD @@ -460,4 +462,3 @@ Although forest conversion emissions are very small in the US, they will become - Potapov, P., Hansen, M.C., Pickens, A. et al. (2022). The Global 2000–2020 Land Cover and Land Use Change Dataset Derived From the Landsat Archive: First Results. Frontiers in Remote Sensing 3, 856903. https://doi.org/10.3389/frsen.2022.856903 - Sanderman, J., Hengl, T. & Fiske, G.J. (2017). Soil carbon debt of 12,000 years of human land use. Proceedings of the National Academy of Sciences 114(36), 9575–9580. https://doi.org/10.1073/pnas.1706103114 - Spawn, S.A., Lark, T.J. & Gibbs, H.K. (2019). Carbon emissions from cropland expansion in the United States. Environmental Research Letters 14, 045009. https://doi.org/10.1088/1748-9326/ab0399 - diff --git a/specs/peatland_methodology_supplement.md b/specs/peatland_methodology_supplement.md index ac56ca9..1d7a749 100644 --- a/specs/peatland_methodology_supplement.md +++ b/specs/peatland_methodology_supplement.md @@ -10,7 +10,7 @@ Other providers instead fold peat drainage into their land use change (LUC) fram At least one provider has built a hybrid approach, where peatland drained within the last 20 years has emissions treated as LUC, while older peatland is treated as LM. This seems a promising variation, but without further refinement, it results in attributing _higher_ emissions to long-drained peatland than more recently drained areas under the GHGP discounting framework, a clearly erroneous outcome. -This document presents an alternative two-part model for estimating peatland drainage emissions within the GHGP/LSRS framework as a _combination_ of an initial pulse of land use change emissions for the 20 years after conversion, followed by steady state land management emissions that persist across decades. This can be seen as an evolution of the current state-of-the-art hybrid approach. However *both* types of emissions are attributed to all peatlands, with their relative importance shifting as time passes from initial drainage. +This document presents an alternative two-part model for estimating peatland drainage emissions within the GHGP/LSRS framework as a _combination_ of an initial pulse of land use change emissions for the 20 years after conversion, followed by steady state land management emissions that persist across decades. This can be seen as an evolution of the current state-of-the-art hybrid approach. However *both* types of emissions are attributed to all peatlands, with their relative importance shifting as time passes from initial drainage. The resulting model is grounded in the physical evidence of how peat carbon behaves after drainage, while integrating directly into the GHGP accounting framework. @@ -22,7 +22,7 @@ Peatlands accumulate organic matter over millennia under waterlogged, anaerobic This carbon store is maintained by high water levels. When the water table is at or near the surface, oxygen cannot penetrate the peat, and decomposition proceeds slowly — limited to anaerobic pathways that produce methane but preserve the bulk of the organic matter. When peatland is converted to agricultural usage, it's systematically drained, setting off a number of GHG release pathways: -- **Gaseous CO₂, CH₄, and N₂O from peat oxidation** is the dominant effect: lowering the water table exposes the upper peat profile to oxygen, activating aerobic microbial decomposition and converting stored carbon in the drained zone above the water table to CO₂ in the air. Nitrogen locked in peat organic matter is likewise broken down and made available for nitrification and denitrification. +- **Gaseous CO₂, CH₄, and N₂O from peat oxidation** is the dominant effect: lowering the water table exposes the upper peat profile to oxygen, activating aerobic microbial decomposition and converting stored carbon in the drained zone above the water table to CO₂ in the air. Nitrogen locked in peat organic matter is likewise broken down and made available for nitrification and denitrification. - **Dissolved organic carbon exported via drainage water** is the second major release pathway. Lowering peatland water tables is not a one-time event: drainage is maintained through a complex series of ditches. As rainwater repeatedly flows through the peat matrix, it flushes carbon rich organic compounds into the water system. The majority subsequently decays to CO₂ in downstream waterways. Evans et al., 2016 estimated that dissolved carbon contributes up to 25% of total peatland carbon fluxes. - **CH₄ emissions from drainage ditches** are a third source. While peatland drainage reduces CH₄ emissions from the peat surface (by eliminating anaerobic conditions), the ditch network creates new anaerobic zones that emit CH₄ at significant rates (IPCC, 2014). - **Finally, peatland fires** are an episodic but catastrophic pathway, particularly in tropical peatlands. Drained peat is highly flammable, and fire events can release more carbon in days than annual oxidative losses produce in years. Peatland fires are excluded from the current methodology, and are an area for future improvement. @@ -43,7 +43,7 @@ Although the scientific literature is clear on the general pathways and time-ser Two principal methods are used to quantify carbon losses from drained peatlands, and they systematically produce different estimates. -**Flux measurements** (eddy covariance towers and closed chambers) directly measure gaseous exchange between the peat surface and the atmosphere. Eddy covariance provides continuous, landscape-scale data; chambers provide spatially targeted but temporally sparse measurements. Both include root respiration (not a peat carbon loss, must be somehow estimated and subtracted out) alongside the peat decomposition signal and exclude waterborne carbon losses. +**Flux measurements** (eddy covariance towers and closed chambers) directly measure gaseous exchange between the peat surface and the atmosphere. Eddy covariance provides continuous, landscape-scale data; chambers provide spatially targeted but temporally sparse measurements. Both include root respiration (not a peat carbon loss, must be somehow estimated and subtracted out) alongside the peat decomposition signal and exclude waterborne carbon losses. **Subsidence monitoring** measures the physical lowering of the peat surface over time and converts it to carbon loss using measured bulk density and carbon content. It integrates oxidative decomposition and dissolved organic carbon export, but can be confounded by physical compaction which lowers the peat surface without exporting CO₂, especially in the first years after drainage. @@ -51,11 +51,11 @@ The result of the different boundaries from the two basic measurement approaches ### IPCC Tier 1 emission factors -The IPCC 2013 Wetlands Supplement made an attempt to consolidate all the best evidence on peatland emissions to that point, and provided Tier 1 default emission factors for drained organic soils, stratified by climate zone and post-drainage land use. +The IPCC 2013 Wetlands Supplement made an attempt to consolidate all the best evidence on peatland emissions to that point, and provided Tier 1 default emission factors for drained organic soils, stratified by climate zone and post-drainage land use. -The table below shows these emissions factors. +The table below shows these emissions factors. -| Climate zone | Land use | CO₂ oxidation | Dissolved carbon | CH₄ | N₂O | **Total** | +| Climate zone | Land use | CO₂ oxidation | Dissolved carbon | CH₄ | N₂O | **Total** | |---|---|---|---|---|---|---| | **Tropical** | Acacia | 73.3 | 3.0 | 1.3 | 1.0 | **78.6** | | | Cropland | 51.3 | 3.0 | 1.4 | 2.1 | **57.9** | @@ -81,7 +81,7 @@ Two studies have attempted to model peatland emission rate declines over multi-d E_DNDC(t) = Σᵢ Aᵢ × exp(-kᵢ × t) for i = each of 4 pools ``` -**Qiu et al. (2021)** used the ORCHIDEE-PEAT land surface model to simulate historical carbon emissions from cultivated northern peatlands. ORCHIDEE also models soil carbon decomposition based on first-order kinetics and turnover times, however, using three carbon pools instead of four (active, slow, passive). +**Qiu et al. (2021)** used the ORCHIDEE-PEAT land surface model to simulate historical carbon emissions from cultivated northern peatlands. ORCHIDEE also models soil carbon decomposition based on first-order kinetics and turnover times, however, using three carbon pools instead of four (active, slow, passive). ## A simplified model for GHGP reporting @@ -107,19 +107,19 @@ E_CO₂(t) = A_fast × exp(-k_fast × t) + A_slow × exp(-k_slow × t) + C This equation approximates the three pool model from Qiu et al. (2021) (active, slow, passive). We fit the double exponential plus constant to Figure S11B from the paper, which plots their modeled emissions decay curve for peatlands converted in 1900. We then repeat this process for Fig. 6 from Swails et al. (2022), which shows modeled CO₂ for the first 30 years of tropical palm plantations. -Notably, the Swails et al. modeled tropical emissions curve is *lower* than the Qiu et al. temperate emissions curve at all years. This is surprising: there is a theoretical basis to suspect that emissions rates from peat should be higher in tropical regions than temperate and boreal regions — decay is highly temperature dependent, with most models suggesting organic compounds should degrade to CO₂ at least 2x faster for every 10 degrees of average temperature increase. The reversal of the expected relationship in the Qiu and Swails studies is partially explained by the fact that the Qiu curve represents gross peat decomposition, while the Swails curve represents net emissions (after vegetation carbon offsets). But this provides only a partial explanation. +Notably, the Swails et al. modeled tropical emissions curve is *lower* than the Qiu et al. temperate emissions curve at all years. This is surprising: there is a theoretical basis to suspect that emissions rates from peat should be higher in tropical regions than temperate and boreal regions — decay is highly temperature dependent, with most models suggesting organic compounds should degrade to CO₂ at least 2x faster for every 10 degrees of average temperature increase. The reversal of the expected relationship in the Qiu and Swails studies is partially explained by the fact that the Qiu curve represents gross peat decomposition, while the Swails curve represents net emissions (after vegetation carbon offsets). But this provides only a partial explanation. The IPCC reference values for CO₂ emissions show a similar pattern. Although the absolute IPCC factors are higher for tropical regions, the difference is no larger than the gap that would be expected from age differences alone, if the general shape of the Swails and Qiu curves is correct. In other words, the IPCC data points also fail to show higher tropical emissions, once corrected for age of the measured sites. Given these observations, we do not attempt to build regional or crop specific curves. Rather we simply use a blended version of the Swails model, the Qiu model, and the IPCC values to construct a single reference CO₂ emissions curve for all climate zones and land use types. We generate this curve by: - adjusting the Swails value to match IPCC tropical palm EF at year 10 (the approximate midpoint of the sites used to generate that value); we leave the Qiu curve as is, since it's already reasonably well calibrated to the IPCC values -- build a new dataset that includes the averages of the Qiu and recalibrated Swails curves for years 1 through 20, and the IPCC temperate/boreal cropland CO₂ value (29.0 t CO₂ ha⁻¹ yr⁻¹) for steady state in the out years, -- fitting a new double exponential to the resulting data points +- build a new dataset that includes the averages of the Qiu and recalibrated Swails curves for years 1 through 20, and the IPCC temperate/boreal cropland CO₂ value (29.0 t CO₂ ha⁻¹ yr⁻¹) for steady state in the out years, +- fitting a new double exponential to the resulting data points We use the cropland value for the steady state because recent literature has undermined the evidence that long-drained peatland has lower emissions on grassland (Holzknecht et al. 2025, Keck et al. 2024); as between the cropland and grassland values, the more recent studies better support the IPCC's cropland EF (Tiemeyer et al., 2020); and in any event this approach ensures the model errs on the side of conservatism. Admittedly, the final single, cross-region, cross-crop curve may be an oversimplification, but we do not see sufficient data to support a more varied approach at this time. This could be an area for future iteration. -Relative to the IPCC values, our final blended curve closely matches the IPCC value for oil palm in the relevant time period, is slightly below the IPCC values for tropical acacia and cropland, and then matches the long-term cropland values for boreal and temperate locations. Notably, the IPCC acacia value — the largest outlier — has been questioned in the more recent literature because it is based on subsidence measurements in extremely young plantations, which are disproportionately inflated by compaction. (Deshmukh et al., 2023). +Relative to the IPCC values, our final blended curve closely matches the IPCC value for oil palm in the relevant time period, is slightly below the IPCC values for tropical acacia and cropland, and then matches the long-term cropland values for boreal and temperate locations. Notably, the IPCC acacia value — the largest outlier — has been questioned in the more recent literature because it is based on subsidence measurements in extremely young plantations, which are disproportionately inflated by compaction. (Deshmukh et al., 2023). The figure below shows the Swails and Qiu curves, the IPCC emissions factors along with the approximate ages of the sites they represent (based on a partial review of the citations in the Wetlands Supplement), and the final blended reference curve. @@ -139,7 +139,7 @@ Next, we add non-CO₂ pathways — dissolved organic carbon (DOC), ditch CH₄, | Boreal | Cropland | 7.6 | | Boreal | Pasture | 6.1 | -As in the previous section, we construct a time-varying non-CO₂ curve. In this case, we just use the IPCC sites directly as a smooth blend between the early acacia values and the late cropland values. +As in the previous section, we construct a time-varying non-CO₂ curve. In this case, we just use the IPCC sites directly as a smooth blend between the early acacia values and the late cropland values. - **Years 1–10:** Non-CO₂ ≈ 5.3 t CO₂-eq ha⁻¹ yr⁻¹ (tropical acacia value, reflecting young drainage) - **Years 40+:** Non-CO₂ ≈ 8.3 t CO₂-eq ha⁻¹ yr⁻¹ (temperate cropland value, reflecting mature drainage) @@ -155,7 +155,7 @@ Note that the IPCC N₂O emission factors are intended to capture N₂O from min Finally we use the updated reference curve — now including both CO₂ and non-CO₂ emissions — to set the two parameters of our GHGP model. -First we set **E_LM** equal to the steady-state of the all-GHG reference curve: the CO₂ floor plus the long-run non-CO₂ asymptote (temperate cropland value). That is **37.3 t CO₂-eq ha⁻¹ yr⁻¹**. +First we set **E_LM** equal to the steady-state of the all-GHG reference curve: the CO₂ floor plus the long-run non-CO₂ asymptote (temperate cropland value). That is **37.3 t CO₂-eq ha⁻¹ yr⁻¹**. Second, we set **P_LUC** by least-squares fit of the GHGP linear ramp to the all-GHG reference curve over years 1 to 20, with E_LM fixed from the previous step. That comes to **621 t CO₂ ha⁻¹**. @@ -167,11 +167,11 @@ The linear ramp undershoots the reference curve in year 1 and slightly overshoot ## Calculations -See the [accompanying python notebook](../notebooks/peatland_emissions_modeling.ipynb) for curve fits and related calculations. +See the [accompanying python notebook](analyses/peatland_emissions_modeling.ipynb) for curve fits and related calculations. ## Appendix: IPCC Literature Review -This catalogues a partial review of IPCC citations. We believe these to be representative (the IPCC Wetlands Supplement acknowledges the age difference between sites used for the published EFs), but completing this review is an area for refinement. +This catalogues a partial review of IPCC citations. We believe these to be representative (the IPCC Wetlands Supplement acknowledges the age difference between sites used for the published EFs), but completing this review is an area for refinement. | IPCC Category | Papers reviewed | Sites with age data | Est. drainage age range | |---|---|---|---| @@ -196,7 +196,7 @@ This catalogues a partial review of IPCC citations. We believe these to be repre - IPCC (2014). 2013 Supplement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories: Wetlands. Hiraishi, T., et al. (eds). IPCC, Switzerland. -- Keck H, Meurer KHE, Jordan S, Kätterer T, Hadden D and Grelle A (2024) Setting-aside cropland did not reduce greenhouse gas emissions from a drained peat soil in Sweden. Front. Environ. Sci. 12:1386134. +- Keck H, Meurer KHE, Jordan S, Kätterer T, Hadden D and Grelle A (2024) Setting-aside cropland did not reduce greenhouse gas emissions from a drained peat soil in Sweden. Front. Environ. Sci. 12:1386134. - Leifeld, J., Klein, K. & Wüst-Galley, C. (2018). Peat decomposability in managed organic soils in relation to land use, organic matter composition and temperature. *Biogeosciences*, 15, 703–719. diff --git a/specs/pipeline_tech_design.md b/specs/pipeline_tech_design.md index b751258..7b8a3ae 100644 --- a/specs/pipeline_tech_design.md +++ b/specs/pipeline_tech_design.md @@ -1,556 +1,185 @@ # Cornerstone jdLUC: pipeline technical design -This document describes the technical design for a data pipeline to implement the jdLUC methodology described in methodology.md. +This document describes the technical design for the data pipeline that implements the jdLUC methodology described in methodology.md. -The pipeline follows an **Extract → Transform → Publish** architecture, inspired by [Cornerstone's MRIO pipeline architecture](https://github.com/cornerstone-data/papers/blob/public-review-draft-tech-arch-vision/architecture_vision/Cornerstone_Architecture_Vision.md): +The pipeline is a linear chain of five stages, each consuming the previous stage's output: -- **Extract**: Ingest external datasets into Google Earth Engine (GEE) as rasters and tables. -- **Transform**: The main computation pipeline; builds a time series of land use transitions, accompanying emissions, and summary tables. -- **Publish**: Publish outputs to BigQuery for downstream queries. +- **Ingest** — download every external source dataset into GCS as tiled GeoTIFFs (raster) or FlatGeobuf (vector). +- **Harmonize** — mosaic the ingested tiles and warp them onto a single common grid, yielding one `xarray.Dataset`. +- **Emissions** — compute per-pixel emissions as a lazy dask/xarray graph and persist to zarr. +- **Attribution** — clip per-pixel emissions to jurisdiction polygons and crop masks, rolling up to per-(jurisdiction, crop) totals. +- **Emissions factors** — join the attribution rollups to NASS yields to produce the final emissions-factor table. -## Platform +There is no Earth Engine and no BigQuery: all raster work runs locally (or on any dask-capable host) over xarray/rasterio, and all outputs land in GCS as zarr and parquet. -All raster processing runs on Google Earth Engine via the `earthengine-api` Python package. GEE provides native raster algebra optimized for planetary-scale compute, and many of the input datasets (GLAD GLC, CDL, Harris AGB, SoilGrids, etc.) are already in its catalog. +## Platform -GEE uses a lazy evaluation architecture: operations on `Image`, `ImageCollection`, and `FeatureCollection` objects build server-side computation graphs. Per-pixel logic is composed from raster algebra functions (`.add()`, `.multiply()`, etc.), conditionals (`.where()`), and masking (`.mask()`, `.updateMask()`). Multiple bands can be stacked into a single target image via `.addBands()`. Computation actually runs when results are materialized with functions like `.getInfo()` (synchronously retrieve computed results), `.evaluate()` (asynchronously retrieve computed results), `Export.image.toAsset()` / `Export.table.toAsset()` (asynchronously materialize a raster or table as a persistent GEE asset), and `Export.image.toDrive()` (asynchronously export a raster to Google Drive as a GeoTIFF or other format). +Raster computation is expressed as lazy `xarray.Dataset` / `dask.array` graphs built with rioxarray and rasterio; GDAL VRTs do the mosaicking and grid warping. Nothing materializes until a stage writes its result, at which point dask streams the computation tile-by-tile into the output store. -The pipeline takes advantage of GEE's lazy evaluation approach to construct the main stages of the computation graph as raster algebra, then materializes results at chosen stage boundaries where we want to persist fixed assets. +Configuration comes from a `.env` file, loaded once via `config.Config.from_dot_env()`: -## Geospatial grid +- `INGEST_BUCKET_NAME` — GCS bucket for ingested source tiles. +- `SCRATCH_BUCKET_NAME` — GCS bucket for cached stage outputs (zarr / parquet). +- `GCP_PROJECT` — project for GCS billing and access. +- `USDA_NASS_API_KEY` — key for the NASS QuickStats yields API. -All raster computation uses the GLAD GLCLUC native grid, pinned as `GLAD_CRS_TRANSFORM = [0.00025, 0, -180, 0, -0.00025, 90]` (global `EPSG:4326`). Every `Export.image.toAsset()` call in the pipeline passes `crs='EPSG:4326'` and `crsTransform=GLAD_CRS_TRANSFORM`. We're extremely explicit because GEE's implicit projection resolution is famously unpredictable when inputs have different native grids — the output origin is inherited from somewhere in the computation graph, and which input wins is brittle to reason about. See GEE's [Projections guide](https://developers.google.com/earth-engine/guides/projections). +GCS access uses application-default credentials (`gcloud auth application-default login`). -Inputs land on this grid at one of two points: +## Geospatial grid -- **Extract-time** for non-GEE-native datasets — i.e. reprojected during upload; -- **Transform-time** for GEE-native datasets — read from the GEE catalog and reprojected in the computation graph +All rasters are harmonized onto the GLAD GLCLUC native grid: global `EPSG:4326` at 0.00025° (~30 m). `harmonize.Grid` constructs this grid from the requested tile set, with `GLAD_TILE_RESOLUTION = XY(40_000, 40_000)` pixels per 10° tile (i.e. 4,000 px/degree → 0.00025°). -Reprojections use nearest-neighbor interpolation for categorical data types and bilinear interpolation for continuous data types. +Harmonization happens in two GDAL steps per source band (`harmonize.get_vrt_for_dataset_band_tile_ids`): -## Architecture +1. **Mosaic** — the ingested 10° tiles are assembled into a `.mosaic.vrt` whose `SimpleSource` entries place each tile at its computed offset on the common grid. Whole-world datasets (e.g. IPCC climate zones) are placed via a single windowed source instead of per-tile. +2. **Warp** — a `rasterio.vrt.WarpedVRT` pins the mosaic to `EPSG:4326` at the grid's transform/resolution, written out as a `.warped.vrt`. -**`pipeline.py`** — the overall entry point; orchestrates the pipeline; +Resampling is a per-dataset property (`RasterDataset.resampling`): nearest-neighbor for categorical layers (GLAD land classes, CDL, peatland mask), bilinear for continuous layers (SoilGrids, Huang BGB). `harmonize.get_dset_for_output` opens the warped VRTs with rioxarray and assembles them into one `xarray.Dataset`, one variable per fully-qualified source band (`{source}:{product}:{band}`), with dtype and no-data unified via `utils.unify_dtype_and_no_data`. -**`extract/`** — makes all input datasets available as GEE assets. +## Caching and versioning -- `extract.py` — extract stage entry point; orchestrates extraction across datasets; -- `[dataset_name].py` — one module per external dataset; each module exposes a function that downloads the dataset, does any minimally necessary preprocessing (e.g. merging tiles to a single raster), and uploads the data as a GEE asset -- GEE-native datasets are used directly: GLAD GLC, CDL, SoilGrids SOC, TIGER state boundaries. +Stage outputs are cached in GCS by the `@gcs.cache(version: int)` decorator. The cache key is a SHA1 over the function's module path, qualified name, the integer `version`, and the call arguments; the decorator picks a serializer by the function's return annotation: -**`transform/`** — computes emissions from GEE assets at region scope. +- `xarray.Dataset` → **`ZarrCacher`** (writes a `.zarr` store under the scratch bucket). +- `pandas.DataFrame` → **`ParquetCacher`**. -- `transform.py` — transform stage entry point; orchestrates the four sequential exports per pipeline run (one task per stage, no per-state subdivision). -- `land_use.py` — builds the `land_use` raster from GLAD GLC + CDL + GFW Peatlands for the requested region geometry. -- `emissions.py` — computes per-pixel emissions from transitions and peatland management for the same region. -- `summary_tables.py` — reduces the raster assets to `transitions` and `crops` tables via a single `reduceRegions` over the region's counties; output FeatureCollections stay server-side from `reduceRegions` through `Export.table.toAsset`. +On a cache hit the stage deserializes the stored result; on a miss it runs and serializes. Bumping the `version` int (or changing any argument) invalidates the entry. Current versions: `harmonize.workflow` v0, `emissions.workflow` v2, `attribution.workflow` v1, `emissions_factors.workflow` v0. -**`publish/`** — make pipeline outputs accessible to consumers. +Provenance is tracked at the ingest layer instead: each ingested GeoTIFF carries metadata tags (`watershed-data-version` = the dataset's declared version, `watershed-processing-version` = git SHA, `watershed-processing-time`, `watershed-source-name`, `watershed-product-name`, `watershed-remote-url`), and tiles are stored under a deterministic prefix `{source}/{product}/{version}/{partitioning}/{tile_id}` so a given source version is addressable and re-fetchable. -- `publish.py`: publish stage entry point; orchestrates publish jobs -- `bigquery.py`: exports the `transitions` and `crops` tables to BigQuery for easy SQL querying. +## Architecture -**`utils/`** +Modules live flat under `jdluc/`: -- `constants.py` — GEE asset IDs, land use codes, IPCC tables, crop groups, etc. -- `gee.py` — GEE initialization -- `states.py` — state boundary helpers -- `transitions.py` — land use transition encoding/decoding helpers -- `version.py` — pipeline versioning from git state. -- `asset_management.py` — listing, version-parsing, and safe-deletion helpers for GEE assets +- **`ingest.py`** — ingest-stage entry point (`python -m jdluc.ingest `). +- **`datasets/`** — one module per source dataset, each exposing a `RasterDataset`, `VectorDataset`, or `TabularDataset` (defined in `datasets/base.py`) plus any source-specific download/preprocess logic. +- **`harmonize.py`** — mosaic + warp into a common-grid `xarray.Dataset`. +- **`emissions.py`** — per-pixel emissions → zarr. +- **`attribution.py`** — per-(jurisdiction, crop) rollups. +- **`emissions_factors.py`** — final EF table. +- **`config.py`, `gcs.py`, `tiling.py`, `utils.py`** — shared infrastructure (config; GCS I/O + content-addressed cache via `@gcs.cache`; tile sets / partitionings / tile-id helpers; VRT / zarr / misc helpers). The dataset base classes (`RasterDataset` / `VectorDataset` / `TabularDataset`) and `AssetType` live in `datasets/base.py`. ### Layering enforcement -The four pipeline packages form a strict dependency chain — `extract → transform → publish → pipeline` — enforced by [import-linter](https://import-linter.readthedocs.io/) contracts. Higher layers may import lower; never the reverse. `jdluc.utils` is shared and every layer is free to import from it. - -## Outputs - -### Rasters - -Two regional rasters per pipeline run: - -#### `land_use_{region}_{version}` - - -| Bands | Count | Type | Description | -| ----------------------------------------------------------------- | ----- | ----- | ----------------------------------------------- | -| `transitions_2000_2005`,`…_2005_2010`,`…_2010_2015`,`…_2015_2020` | 4 | uint8 | Land use transition per GLAD GLC epoch boundary | -| `crops_2020` | 1 | uint8 | CDL crop code for GLAD cropland pixels, 2020 | -| `is_peatland` | 1 | uint8 | peatland mask | - - -The `transitions_{epoch}` bands preserve the full 9×9 space of land use category transition pairs, including nonemissive ones. `0` denotes "no transition" (same category at both epoch boundaries). See `utils/transitions.py` for the `uint8` encoding scheme for transition pairs. - -#### `emissions_{region}_{version}` - -LUC and peatland emissions per epoch transition + 2020-allocated values - - -| Bands | Count | Type | Description | -| ---------------------------------------------------------------------------- | ----- | ------- | --------------------------------------------------------------------------------------------- | -| `luc_emissions_2000_2005`, `…_2005_2010`, `…_2010_2015`, `…_2015_2020` | 4 | float32 | Forest + short vegetation + SOC emissions per epoch transition (no peatland), tCO2e per pixel | -| `peatland_conversion_2000_2005`, `…_2005_2010`, `…_2010_2015`, `…_2015_2020` | 4 | float32 | Peatland transformation (land use change) emissions per epoch transition, tCO2e per pixel | -| `peatland_occupation_2020` | 1 | float32 | Annual peatland occupation (land management) emissions, tCO2e per pixel | -| `allocated_luc_emissions_2020` | 1 | float32 | 20-year weighted sum of LUC transition emissions, tCO2e per pixel | -| `allocated_peatland_emissions_2020` | 1 | float32 | 20-year weighted peatland conversion emissions + occupation, tCO2e per pixel | - - -### Tables - -#### `transitions_{region}_{version}` - -Grain: one row per `(county_fips, epoch_transition, emissions_type)`. - - -| Column | Type | Description | -| ----------------------------- | ------- | ------------------------------------------------------ | -| county_fips | STRING | 5-digit combined state+county FIPS code | -| epoch_transition | STRING | Epoch transition (e.g. "2000_2005", "2015_2020") | -| emissions_type | STRING | One of the types below | -| total_area_ha | FLOAT64 | Total land area affected by this transition (hectares) | -| total_emissions_tco2 | FLOAT64 | Emissions from this transition | -| allocated_emissions_2020_tco2 | FLOAT64 | 2020-allocated emissions | - -**`emissions_type` values:** - -- **LUC transition types**: `forest_to_cropland`, `forest_to_built_up`, `forest_to_short_veg`, `wetland_forest_to_cropland`, `wetland_forest_to_built_up`, `short_veg_to_cropland`, `short_veg_to_built_up`, `wetland_short_veg_to_cropland`, `wetland_short_veg_to_built_up` — carry LUC-only emissions (forest + short vegetation + SOC, no peatland). Note: GLAD GLC does not distinguish pastureland from grassland, so there are no separate pastureland transition types. -- **`peatland_conversion`**: peatland transformation emissions on transition pixels, one row per `(county_fips, epoch_transition)` -- **`peatland_occupation`**: annual peatland occupation emissions for epoch_transition=2020 only, on all peatland pixels under cultivation. One row per county. `allocated_emissions_2020_tco2 = total_emissions_tco2` for these rows. - -#### `crops_{region}_{version}` - -Grain: one row per `(county_fips, crop_code)`. Yields are applied uniformly from state-level NASS data (see known issues in methodology.md); county-level emissions factors still vary because per-hectare emissions vary across counties. - - -| Column | Type | Description | -| ------------------------------ | ------- | ---------------------------------------------------------------------------------------------------- | -| county_fips | STRING | 5-digit combined state+county FIPS code | -| crop_code | INT64 | CDL code | -| crop_group | STRING | 'corn', 'soybeans', 'wheat' | -| total_production_kg | FLOAT64 | County 2020 production in kg (county area × state-averaged yield) | -| total_production_bu | FLOAT64 | County 2020 production in bushels (county area × state-averaged yield) | -| total_crop_area_ha | FLOAT64 | Total ha of this crop in county (2020) | -| peatland_crop_area_ha | FLOAT64 | Ha of this crop on peatland in county (2020) | -| yield_kg_per_ha | FLOAT64 | NASS state-level yield rate (4-year mean, 2017-2020); applied uniformly to all counties in the state | -| yield_bu_per_acre | FLOAT64 | NASS state-level yield rate (native unit); applied uniformly to all counties in the state | -| total_allocated_emissions_tco2 | FLOAT64 | Sum of allocated_emissions_2020_tco2 across all emissions types | -| emissions_factor_kgco2e_per_kg | FLOAT64 | allocated / production × 1000 | -| pct_forest | FLOAT64 | % of allocated from forest transitions | -| pct_short_veg | FLOAT64 | % from short vegetation transitions | -| pct_peatland_conversion | FLOAT64 | % from peatland conversion | -| pct_peatland_occupation | FLOAT64 | % from peatland occupation | -| pct_epoch_2005…pct_epoch_2020 | FLOAT64 | % of allocated by epoch transition | - - -## Stage Details - -### Overall entry point - -The overall pipeline entry point is `run_pipeline()` in `pipeline.py`. - -#### `run_pipeline(gcp_project, states, region_name, force=False) -> PipelineResult` - -Computes and publishes assets for the states defined in the `states` array, running the `extract/`, `transform/`, and `publish/` steps in sequence. `states` can also take the special value `['CONUS']` to compute outputs for all 48 continental US states + the District of Columbia. `region_name` is the identifier used in the regional raster and table asset names. - -On success, returns a `PipelineResult` dataclass with the pipeline version, region name, materialized asset IDs (or `ee.Image` / `ee.FeatureCollection` handles) from each stage, and a `from_cache` flag indicating whether any work was done on this invocation. On failure, raises the originating exception after logging to stderr. - -Each stage checks for previously-computed assets at its expected version and reuses them on cache hit, so re-runs after partial failures avoid recomputing completed work. Certain parts of the pipeline have very simple retry logic, but transient failures (GEE quota limits, timeouts, etc.) are mostly handled by simply re-running the pipeline. - -### Extract - -The Extract stage makes all input datasets available as GEE assets. - -#### Entry point - -The entry point to the extract steps is `extract_all(gcp_project, force=False)` in `extract.py`. - -`extract_all()` begins by loading the dataset inventory stored in `utils/constants.py`: - - -| Dataset | Source | GEE Asset ID | Resolution | GEE-native? | -| ------------------------ | ---------------------------------------------- | --------------------------------------------------------------------- | --------------------- | ----------- | -| GLAD GLCLUC v2 | GEE catalog | `projects/glad/GLCLU2020/v2/LCLUC_{year}` | 0.00025° (~30m) | GEE-native | -| USDA CDL | GEE catalog | `USDA/NASS/CDL` | 30m (native Albers) | GEE-native | -| SoilGrids SOC (0–30cm) | GEE catalog | `projects/soilgrids-isric/ocs_mean` | 250m (native IGH) | GEE-native | -| US state boundaries | GEE catalog | `TIGER/2018/States` | Vector | GEE-native | -| Harris et al. (2021) AGB | Global Forest Watch (ArcGIS) | `projects/cornertone-luc/assets/high-res-luc/harris_agb_conus` | 0.00025° (~30m) | No | -| Huang et al. (2021) BGB | Figshare (doi:10.6084/m9.figshare.12199637.v1) | `projects/cornertone-luc/assets/high-res-luc/huang_bgb_conus` | 0.0083° (~1km) | No | -| NASS crop yields | USDA QuickStats | `projects/cornertone-luc/assets/high-res-luc/nass_yields` | State-level (tabular) | No | -| IPCC climate zones | Zenodo (doi:10.5281/zenodo.7303808) | `projects/cornertone-luc/assets/high-res-luc/ipcc_climate_zones` | 0.5° (~50km) | No | -| GFW Global Peatlands | Global Forest Watch Open Data (CC BY 4.0) | `projects/cornertone-luc/assets/high-res-luc/gfw_peatlands_v20230315` | 0.00025° (~30m) | No | - - -All raster pixel sizes are shown as ~meter-equivalent at CONUS mid-latitudes. CDL and SoilGrids have projected native coordinate reference systems (Albers Equal Area and Interrupted Goode Homolosine, respectively), so meters are the primary unit for those two rows; all others are natively on a WGS84 degree grid. - -For each non-GEE-native dataset, `extract_all()` checks whether the target GEE asset already exists, in the expected version. If the asset is missing (or `force=True`), `extract_all()` calls the applicable dataset function to download, convert, and then upload that dataset to GEE. Conversion steps are minimal: just what's necessary to get the dataset into GEE and pinned onto `GLAD_CRS_TRANSFORM` ([Platform](#Platform)). All other computation stays in the transform pipeline. As it completes each dataset, `extract_all()` logs status: cached / extracted / failed. - -#### Asset versioning - -Extract assets are each tagged with a suffix derived from the upstream dataset's version — e.g., `harris_agb_conus_v2021`, `nass_yields_v2017_2020`. The expected version for each dataset is defined in the `constants.py` dataset inventory table. `extract_all()` compares the expected version against what exists in GEE: if the expected asset is missing, it runs the extraction steps. If prior versions of a dataset also exist, `extract_all()` logs a warning but does not delete them. - -Unlike later stages of the pipeline, extract assets are *not* tagged with a SHA of the `extract/` pipeline code version that uploaded them. The extract logic is simple and stable; on the rare occasions it changes, there is a simple utility in `cli/` to flush cached assets. - -To update a source dataset (e.g., extending NASS yields through 2024), update the expected version in `constants.py` and re-run `run_extract`. The old asset remains until manually deleted. - -#### Source-data mirroring - -Every non-native input dataset (Harris AGB, Huang BGB, NASS QuickStats, IPCC climate zones, GFW Peatlands) is mirrored to `gs://cornerstone-luc/luc_high_res/extract_mirror/{dataset}/` on first successful upstream fetch. Subsequent extract runs read from the mirror first and fall back to upstream only on mirror miss. All five sources permit mirroring with attribution (Harris CC-BY, Huang CC-BY-4.0, NASS US-gov public domain, IPCC open Zenodo, GFW Peatlands CC-BY-4.0). - -Most pipeline runs are already insulated from upstream availability because the transform stage reads the cached GEE extract asset, not the raw source — if a publisher takes down their site tomorrow, the pipeline keeps working as long as `{dataset}_{version}` exists in GEE. But the mirroring protects against change to extract-stage logic that requires re-deriving the GEE asset from the original source. The extract modules do a small but real amount of processing between the source file and the GEE asset. - -GEE-native assets are **not** mirrored. The reproducibility argument doesn't apply — we don't do any extract-stage processing on them (they're read directly from the GEE catalog by transform), so there's no "re-derive the GEE asset from source" operation to guard. Upstream removal of the catalog asset itself IS a risk, but empirically low: GLAD has retained v1 (`projects/glad/GLCLU2020/LCLUC_{2000,2020}`) years after v2 shipped, indicating stable version stewardship; CDL / TIGER / SoilGrids have institutional backing that makes deprecation risk low. - -#### Dataset details - -None of the following extract steps require API keys or other credentials beyond standard GCP auth (`gcloud auth login`) and GEE auth (`earthengine authenticate`) for the upload side. - -GEE image ingestion requires the source file to live in a GCS bucket first, so the Harris and Huang extractors both include a transient GCS staging step (upload → ingest → delete GCS object). NASS and IPCC avoid this because GEE's table-asset and small-image paths accept direct uploads. - -Tile-based extracts (Harris AGB, GFW Peatlands) fan out their per-tile work into three phases: concurrent download + GCS staging via a thread pool, synchronous batch submission of all ingest tasks, single bulk poll until all reach a terminal state. GEE caps per-project concurrent ingest tasks at ~20 (empirical on the Limited tier), so the wall-time floor is `total_tile_minutes / 20` regardless of how clients submit — sequential per-tile submit-and-wait would leave 19/20 slots idle. Per-tile failures are isolated: a single failed tile logs but doesn't abort siblings; the resulting ImageCollection is partial and can be repaired by re-running with `force=True`. - -##### Harris AGB - -1. **Discover**: Query the Global Forest Watch ArcGIS FeatureServer (`services2.arcgis.com/.../Aboveground_Live_Woody_Biomass_Density/FeatureServer/0/query`) for each target tile ID to retrieve the per-tile pre-signed download URL from the feature's `Mg_ha_1_download` attribute. No API key is required; the FeatureServer is public. -2. **Download**: Fetch the 21 10°×10° GeoTIFF tiles covering CONUS (rows 30N/40N/50N × columns 070W–130W). Units are Mg/ha at ~30m (0.00025°) resolution. -3. **Upload**: Stage tiles in GCS and ingest them into GEE *as separate images in a single `ImageCollection`* at `projects/cornerstone-luc/assets/high-res-luc/harris_agb_conus` (not pre-merged into a single mosaic — GEE handles the mosaicking lazily at read time). - -##### Huang BGB - -1. **Download**: Fetch `data_code_to_submit.zip` (~877 MB) directly from the paper's Figshare deposit (`https://ndownloader.figshare.com/files/22432460`, DOI `10.6084/m9.figshare.12199637.v1`, no auth required) and extract `pergridarea_bgb.nc` from it. -2. **Convert**: Open the NetCDF, select the `AROOT` variable, clip to a CONUS bounding box, tag with EPSG:4326, and write as a LZW-compressed float32 GeoTIFF. -3. **Upload**: Stage the GeoTIFF in GCS and ingest into GEE as an `Image` asset at `projects/cornerstone-luc/assets/high-res-luc/huang_bgb_conus`. - -##### NASS crop yields - -1. **Download**: Fetch the full USDA NASS QuickStats crops file (`https://www.nass.usda.gov/datasets/qs.crops_{YYYYMMDD}.txt.gz`, ~1.5 GB gzipped, no auth required). The filename is time-stamped by release date, so each dataset version corresponds to a different URL — the expected URL is stored in `constants.py` alongside the version string. -2. **Upload**: Ingest the raw filtered records as a GEE `FeatureCollection` table asset at `projects/cornerstone-luc/assets/high-res-luc/nass_yields`, keyed by state centroid. All filtering, unit conversion (bu/acre → kg/ha using 7 CFR 810 bushel weights), and multi-year averaging happens in the transform stage. - -##### IPCC climate zones - -1. **Download**: Fetch the IPCC climate zone raster from Zenodo (DOI: 10.5281/zenodo.7303808, Ogle et al. 2006) as a plain HTTP download; no auth required. -2. **Upload**: Ingest into GEE as an `Image` asset at `projects/cornerstone-luc/assets/high-res-luc/ipcc_climate_zones`. - -##### GFW Peatlands - -1. **Discover**: Tile download URLs are template-constructed from a stable GFW Open Data API key plus `tile_id` (e.g. `40N_080W`) — no per-tile FeatureServer roundtrip. The 19-tile CONUS list (`CONUS_TILE_IDS` in `extract/gfw_peatlands.py`) hard-codes the v20230315 manifest; two of Harris AGB's 21 CONUS tiles (30N_070W, 30N_130W) are absent because GFW omits tiles with zero peatland pixels. -2. **Download**: Fetch the 19 10°×10° GeoTIFF tiles. Each is a uint8 single-band mask at ~30m (0.00025°), already aligned to the Hansen Global Forest Change grid (which matches `GLAD_CRS_TRANSFORM`) — no local reprojection or rasterization needed. -3. **Upload**: Stage each tile in GCS and ingest into GEE as separate images in a single `ImageCollection` at `projects/cornerstone-luc/assets/high-res-luc/gfw_peatlands_v20230315` (mirroring the Harris AGB collection shape; `transform/land_use.py::_load_is_peatland` lazy-mosaics on read). - -GFW publishes Global Peatlands as a 30m composite raster: Xu et al. (2018) PEATMAP above 40°N (the source for CONUS), Gumbricht et al. (2017) elsewhere, with regional overrides for Indonesia/Malaysia (Miettinen 2016), the lowland Peruvian Amazon (Hastie 2022), and the Congo basin (Crezee 2022). - -### Transform - -The Transform stage is the main computation pipeline. It reads from GEE assets (both native catalog datasets and extracted assets from the `extract/` stage), computes per-pixel emissions, and exports results as versioned regional GEE raster and table assets. - -#### Overall stage architecture - -##### Module structure - -**`transform.py`** orchestrates four GEE exports per pipeline run, in three sequential steps, all scoped to the requested region's union geometry: - -1. `land_use.py` exports the `land_use_{region}_{version}` raster. -2. `emissions.py` exports the `emissions_{region}_{version}` raster, reading the materialized `land_use` asset. -3. `summary_tables.py` exports `transitions_{region}_{version}` and `crops_{region}_{version}` in parallel, both via a single server-side `reduceRegions` over the region's TIGER counties. - -A pipeline run targets a "region" which is either an array of `states` or the special 'CONUS' value. The region resolves once to (a) an `ee.Geometry` for raster builds (`get_multi_state_boundary`) and (b) an `ee.FeatureCollection` of counties. Both are passed through to the per-stage builders. - -GEE's auto-tiling handles CONUS-scale single-task exports comfortably, which is how each of the land_use, emissions, transitions, and crops exports are run. We tested single, full-region jobs up to CONUS scale versus sharding large regions into state-level GEE tasks, and the single-region approach is 2-10x faster. GEE limits parallelization at the user task level, whereas internal parallelization within a single task is very high: a full CONUS pipeline run and a Delaware run both take similar amounts of time. - -##### Asset versioning - -Assets are tagged with the pipeline version that produced them. Version strings are computed using `utils/version.py` and appended to file names. - -At runtime, the transform stage checks for an asset at the exact expected version string. If missing, it computes the asset; if present, it reuses the existing assets. - -Prior-version assets are silently ignored (this is unlike the extract stage, which warns about prior versions; transform assets turn over more frequently, so we decided warning on their presence by default would be noisy). - -#### Module details +The stages form a strict dependency chain enforced by an [import-linter](https://import-linter.readthedocs.io/) `layers` contract ("ETL pipeline direction" in `pyproject.toml`), highest to lowest: -##### transform.py (entry point) +``` +emissions_factors → attribution → emissions → harmonize → ingest +``` -`run_transform(gcp_project, states, region_name, force=False) -> TransformResult` is the entry point. +Higher layers may import lower; never the reverse. The shared infrastructure modules (`config.py`, `gcs.py`, `tiling.py`, `utils.py`) and the `datasets/` modules sit outside the chain and may be imported anywhere. -Resolves the region geometry and counties FeatureCollection once, then advances through four global phases: `LAND_USE_PENDING` → `LAND_USE_DONE` → `EMISSIONS_PENDING` → `EMISSIONS_DONE` → `TABLES_PENDING` → `TABLES_DONE` | `FAILED`. The `transitions` and `crops` exports submit in parallel during `TABLES_PENDING`. A polling loop (~30s cadence) drives phase transitions because GEE's batch API lacks callbacks. Each stage's downstream consumer reads the prior stage's materialized asset by ID, not its pre-export DAG. - -On success returns a `TransformResult` dataclass carrying the four asset IDs (`land_use_asset_id`, `emissions_asset_id`, `transitions_table_id`, `crops_table_id`) and the pipeline version. On failure raises `TransformError` with the failed phase and error details. - -##### land_use.py - -Builds the GEE computation graph for land use classification and transition detection across the five GLAD GLC epochs, and exports the result as a versioned raster asset. - -###### `build_land_use_image(geometry) -> EEImage` - -Constructs the 6-band `land_use` raster for the requested region geometry: - -1. Loads GLAD GLCLUC v2 for each of the five epochs (2000, 2005, 2010, 2015, 2020), clipped to `geometry`. -2. Classifies each epoch via `classify_glad_glc`. -3. Calls `detect_transitions` for each consecutive epoch pair, producing the four `transitions_{from}_{to}` bands. -4. Loads CDL 2020, reprojects onto `GLAD_CRS_TRANSFORM` (nearest-neighbor), masks to GLAD 2020 cropland pixels, producing the `crops_2020` band. -5. Loads the PEATMAP (Xu et al. 2018) mask asset, reprojects onto `GLAD_CRS_TRANSFORM` (nearest-neighbor), producing the `is_peatland` band. - -Returns the 6-band image described in the `land_use` raster spec above. Pure graph building — no materialized assets. - -###### `export_land_use_asset(image, region, version, geometry, force) -> ee.batch.Task` - -Submits an `ee.batch.Export.image.toAsset()` job for the image with `region=geometry`, `maxPixels=int(1e13)`. The build graph projects every band onto `GLAD_CRS_TRANSFORM` via explicit `.reproject(...)` inside `build_land_use_image`, so the export omits the `crs` / `crsTransform` keyword arguments — passing them on top of an already-grid-aligned input triggers a no-op-but-not-actually-no-op reprojection. Asset ID follows `land_use_{region}_{version}`. Returns the task handle for `transform.py` to poll. - -###### `classify_glad_glc(image) -> EEImage` - -Reclassifies raw GLAD GLCLUC pixel values into simplified category codes (defined in `constants.py`): - - -| GLAD GLCLUC pixel values | Land use category | -| ------------------------ | ---------------------------------------------------------- | -| 25–48 | Forest (terra firma tree cover, by canopy height) | -| 125–148 | Wetland forest | -| 1–24 | Short vegetation (grassland, shrubland, sparse vegetation) | -| 100–124 | Wetland short vegetation | -| 244 | Cropland | -| 250 | Built-up | -| 200–207 | Water | -| 241 | Snow/ice | -| 0 | Bare | - - -Category codes are single-digit integers (1–9), stored as uint8. - -###### `detect_transitions(classified_from, classified_to) -> EEImage` - -Compares two classified epoch images pixel-by-pixel and returns a single uint8 band encoding the transition type, produced via `utils/transitions.encode_transition()`. Pixels where the category is unchanged between epochs are set to 0. See `utils/transitions.py` for the encoding scheme. - -##### emissions.py - -Builds the GEE computation graph for per-pixel emissions from a cached `land_use` asset, and exports the result as a versioned raster asset. - -###### `build_emissions_image(land_use_asset_id, geometry) -> EEImage` - -Constructs the 11-band `emissions` raster for the requested region geometry. Takes the asset ID of the cached `land_use` raster — not an in-memory image — and loads it via `ee.Image(land_use_asset_id)`. This ensures emissions are computed against the finalized, exported land use raster rather than re-deriving land use from raw GLAD inputs at emissions-export time. - -Loads all emissions input datasets (Harris AGB, Huang BGB, SoilGrids SOC, IPCC climate zones, GFW Peatlands) from their GEE assets, clipped to `geometry`. SoilGrids and Huang BGB are reprojected onto `GLAD_CRS_TRANSFORM` at read time via explicit `.reproject()` (bilinear for both continuous layers); Harris AGB, IPCC climate zones, and GFW Peatlands arrive pre-aligned from extract — they were rasterized to the GLAD grid at ingest time. Then: - -1. Iterates over the four epoch pairs, calling `calculate_epoch_emissions` to produce `luc_emissions_{epoch}` and `peatland_conversion_{epoch}` bands. -2. Calls `calculate_peatland_occupation` to produce the single `peatland_occupation_2020` band. -3. Calls `calculate_allocated_emissions_2020` to produce the two `allocated_*_2020` bands. - -Stacks into the 11-band image described in the `emissions` raster spec above. Pure graph building — no exports. - -###### `export_emissions_asset(image, region, version, geometry, force) -> ee.batch.Task` - -Submits an `ee.batch.Export.image.toAsset()` job for the image with `region=geometry`, `maxPixels=int(1e13)`, and `crs` / `crsTransform` omitted (the input is already on `GLAD_CRS_TRANSFORM` via the per-band `.reproject(...)` inside `build_emissions_image`). Asset ID follows `emissions_{region}_{version}`. Returns the task handle. - -###### `calculate_epoch_emissions(transition_band, inputs) -> EEImage` - -Takes a single epoch's encoded transition band plus the pre-loaded input datasets, and returns a two-band image: `luc_emissions` and `peatland_conversion`. - -Decodes the transition band into from/to code bands via the helpers in `utils/transitions.py`, then computes LUC emissions via chained `.where()` expressions keyed on the decoded from/to pair — a single logical pass over the raster that handles all transition types simultaneously, no per-type loop. For each emissive transition, the value is the sum of applicable carbon stock deltas: - -- **Forest → lower-carbon category**: AGB + BGB + DOM loss from forest stock, plus SOC delta weighted by the pixel's IPCC climate zone factor -- **Short vegetation → lower-carbon category**: IPCC grassland carbon stock loss, plus climate-weighted SOC delta -- **Non-emissive transitions** (to-higher-carbon or inert pairs like forest → water): zero - -`peatland_conversion` adds the peatland transformation component on pixels where `is_peatland AND transition is emissive`, using the IPCC peatland conversion emissions factor. - -###### `calculate_peatland_occupation(land_use_image, inputs) -> EEImage` - -Returns a single-band image: annual peatland occupation (land-management) emissions for 2020. Masked to pixels that are both cropland (per GLAD 2020) and peatland. Value is the IPCC peatland cultivation emissions factor (`PEATLAND_EF_TCO2E_HA_YR`) multiplied by pixel area. Independent of whether a conversion transition occurred in the analysis window. - -###### `calculate_allocated_emissions_2020(luc_epoch_bands, peatland_epoch_bands, occupation_band) -> EEImage` - -Applies GHGP 20-year lookback weighting to produce two bands: - -- `allocated_luc_emissions_2020`: weighted sum across the four `luc_emissions_{epoch}` bands -- `allocated_peatland_emissions_2020`: weighted sum across the four `peatland_conversion_{epoch}` bands, plus the `peatland_occupation_2020` annual value at full weight - -Epoch weights follow the linear discount formula in the methodology, using the midpoint year of each epoch window relative to 2020. - -##### summary_tables.py - -`compute_region_tables(land_use_asset, emissions_asset, region, version, counties, force=False) -> dict` - -Top-level entry point for region-scope table computation. Checks for cached `transitions` and `crops` table assets; if both exist, returns their paths. Otherwise builds and exports both via a single server-side `reduceRegions` call per table. Returns a dict of asset paths. - -The aggregation FeatureCollection stays server-side end-to-end: `reduceRegions(...)` returns an `ee.FeatureCollection`, `.map(...)` chains attach derived columns, `.merge(...)` combines per-reducer outputs, and `Export.table.toAsset` writes the result. No `.getInfo()` round-trips during graph construction. Both `reduceRegions` calls pass `tileScale=4` to bound per-tile memory at CONUS scale. - -###### `build_transitions_table(land_use_asset, emissions_asset, counties) -> FeatureCollection` - -Runs one `reduceRegions` per (epoch, emissions-type) combination over `counties`, plus one for peatland occupation. Grain: `(county_fips, epoch_transition, emissions_type)`. Per-reducer outputs are merged into a single `FeatureCollection` server-side. No crop grouping — all transitions are captured regardless of current land use. - -###### `build_crops_table(land_use_asset, emissions_asset, counties) -> FeatureCollection` - -For each epoch transition: - -- masks to pixels that both transitioned and have a 2020 crop code in the `land_use` raster, -- groups by composite key (`type_code × 256 + crop_code`) so all (transition, crop) combinations resolve in a single `reduceRegions` call, -- sums emissions, allocating each pixel to the crop grown on it, summed within county boundaries. - -State-level NASS yields are embedded as an `ee.Dictionary` keyed by `f'{state_fips}|{crop_code}'` (~50 states × 3 crops is small enough to ship as a constant, avoiding an `ee.Join` against a separate asset). The dictionary is looked up via `.map()` over the FeatureCollection to attach `yield_kg_per_ha` and `yield_bu_per_acre` to each county row, producing one row per `(county_fips, crop_code)`. Emissions are divided by production to get final emissions factors. - -### Publish - -The Publish stage makes Transform outputs accessible to consumers. Currently this is BigQuery for downstream queries. - -#### Entry point - -`run_publish(gcp_project, region_name, version, force=False) -> PublishResult` in `publish.py` orchestrates all publish targets. Today this is a thin passthrough to two BigQuery export jobs (one for `transitions`, one for `crops`) — mildly overbuilt for the current scope — but maintaining symmetry with Extract and Transform gives `pipeline.py` a single handle per stage and provides a stable home for future publish targets (tile-serving, GCS exports, report generation, etc.) without further refactoring. +## Outputs -#### BigQuery export (`bigquery.py`) +### Per-pixel emissions (zarr) -Exports the `transitions` and `crops` tables to BigQuery for downstream SQL queries. Reads from the regional `transitions` and `crops` GEE table assets produced by the Transform stage. Both tables carry the same region/transform/publish version triplet so a given pipeline run produces a coherent pair. +`emissions.workflow` returns an `xarray.Dataset` (cached as zarr). Variable names carry unit suffixes via `merge_name_units` (e.g. `emissions-per-hectare:tco2e-per-ha`): -### Utils +| Variable(s) | Description | +|---|---| +| `land-class:{year}` | Per-year `LandClass` code (2000, 2005, 2010, 2015, 2020) | +| `aboveground-carbon`, `belowground-carbon`, `dead-organic-matter-carbon`, `grassland-carbon` | Carbon-stock layers (`:tcarbon-per-ha`) | +| `vegetation-carbon:{year}` | Per-year total vegetation carbon | +| `vegetation-emissions:{b}-{a}`, `soil-emissions:{b}-{a}`, `emissions:{b}-{a}` | Per-epoch-transition fluxes (`tco2e-per-ha`) | +| `peatland-occupation` | Annual peatland occupation (land-management) emissions | +| `emissions-per-hectare` | 20-year linearly-discounted LUC + peatland occupation | +| `hectares-per-pixel` | Pixel area | +| `emissions` | `emissions-per-hectare × hectares-per-pixel` (tCO2e) | -#### constants.py +### Per-(jurisdiction, crop) emissions factors (parquet) -Central home for cross-module constants: GEE asset IDs and expected versions for each input dataset, simplified land use category codes, IPCC emissions factor tables (climate zone × stock change), crop group definitions, and the dataset inventory consumed by `extract_all()`. +`emissions_factors.workflow` returns a `pandas.DataFrame` indexed by `(admin_level, crop_name, jurisdiction_name)`: -#### gee.py +| Column | Description | +|---|---| +| `crop_hectares` | Crop area in the jurisdiction | +| `peatland_crop_hectares` | Crop area on peatland | +| `peatland_occupation_emissions` | Annual peatland-occupation emissions on crop pixels | +| `total_emissions` | Total allocated emissions (tCO2e) | +| `total_production_kg` | `crop_hectares × NASS yield (kg/ha)` | +| `emissions_factor_kgco2e_per_kg` | `total_emissions × 1000 / total_production_kg` | +| `peatland_occupation_fraction` | `peatland_occupation_emissions / total_emissions` | -Provides `initialize_gee(project)`, which authenticates (if credentials aren't already cached) and initializes the Earth Engine client against the high-volume endpoint (`earthengine-highvolume.googleapis.com`) with the given GCP project for billing and asset ownership. The high-volume endpoint is required for the parallel batch-export volume this pipeline generates. +The `attribution.workflow` rollup (same index, columns `crop_hectares`, `peatland_crop_hectares`, `peatland_occupation_emissions`, `total_emissions`) is the EF table's direct input. The legacy `transitions` table has no equivalent — per-transition detail now lives in the per-pixel zarr variables. -Called once per process — by `pipeline.py` at the top of `run_pipeline()`, or by a test fixture when stage entry points (`extract_all`, `run_transform`, `run_publish`) are invoked directly. Stage entry points themselves assume the client is already initialized. +## Stage details -Also exposes the ingest-task primitives used by the extract stage: `start_ingestion_no_wait` and `wait_for_tasks` for tile-based fan-out (see § Extract), and `start_ingestion_and_wait` (a thin wrapper around the two) for single-asset extracts. +### Ingest (`ingest.py`, `datasets/`) -#### states.py +`python -m jdluc.ingest ` (positional; `--concurrency`, `--overwrite` optional). `workflow()` validates that every tile in the set is valid for the dataset's partitioning, then fans the per-tile `dataset.ingest_a_tile(...)` calls out over a thread pool, returning a `{tile_id: result | Exception}` map (per-tile failures are isolated; the process exit code is the failure count). -Provides `get_multi_state_boundary(state_fips_list)` helper, which returns `ee.Geometry` objects from the `TIGER/2018/States` asset. +Each `datasets/` module declares a `RasterDataset` or `VectorDataset`: -#### transitions.py +- **`RasterDataset`** downloads its source, writes a GeoTIFF tagged with provenance metadata, and uploads to the ingest bucket. Bands are namespaced `{source}:{product}:{band}`. +- **`VectorDataset`** stages a vector file, converts it to FlatGeobuf (`utils.convert_vector_to_flatgeobuf`, projecting the configured id/name columns), and uploads. -Encoding and decoding helpers for land use transitions. Transitions are stored as a single uint8 value with the upper 4 bits holding `from_code` and the lower 4 bits holding `to_code`. `0x00` indicates no transition; otherwise the encoded value is `(from_code << 4) | to_code`. For example, forest (1) → cropland (5) encodes as `0x15` (decimal 21). This scheme supports up to 16 land use categories per side — comfortably above the current 9. +Tile sets (`tiling.TileSetName`): `BAY_AREA`, `CONUS`, `DELAWARE`, `GFW`, `WHOLE_WORLD`. Partitionings (`tiling.Partitioning`): `TEN_DEGREE_TILE` (most rasters), `WHOLE_WORLD` (IPCC climate zones), and the vector partitioning for World Bank boundaries. -Provides two forms of each operation — scalar (`int → int`) for lookup-table construction and `.where()` matchers, and raster (`ee.Image → ee.Image`) for the GEE pipeline. The two share a bit layout by construction (both live in this single module). +#### Source dataset inventory -Scalar: +| Dataset (`DatasetName`) | Source | Partitioning | Kind | +|---|---|---|---| +| `GLAD_GLCLUC` | GLAD/Hansen GeoTIFFs (`…/GLCLU2000-2020/v2/{year}/{tile}.tif`) | 10° tile | raster | +| `USDA_NASS_CDL` | USDA Cropland Data Layer | 10° tile | raster | +| `SOILGRIDS_OCS` | ISRIC SoilGrids 0–30 cm OCS (WCS) | 10° tile | raster | +| `GFW_HARRIS_AGB` | GFW data-api (WHRC AGB 2000 v1.4) | 10° tile | raster | +| `HUANG_BGB` | Figshare (doi:10.6084/m9.figshare.12199637) | 10° tile | raster | +| `IPCC_CLIMATE_ZONES` | Zenodo (doi:10.5281/zenodo.7303808) | whole-world | raster | +| `GFW_GLOBAL_PEATLANDS` | GFW data-api (`gfw_peatlands` v20230315) | 10° tile | raster | +| `USDA_NASS_QUICKSTATS` | USDA NASS QuickStats API (yields) | whole-world | tabular | +| `WORLD_BANK_ADMIN_0/1/2` | World Bank Official Boundaries (`.gpkg`, CC BY 4.0) | vector | vector | -- `encode_transition(from_code: int, to_code: int) -> int`: returns `(from_code << 4) | to_code`, or `0` when `from_code == to_code`. -- `decode_from(encoded: int) -> int`: returns `encoded >> 4`. -- `decode_to(encoded: int) -> int`: returns `encoded & 0xF`. +### Harmonize (`harmonize.py`) -Raster: +`python jdluc/harmonize.py `. `workflow(dataset_names, tile_ids)` (`@gcs.cache(version=0)`) builds the common `Grid` from the tile set, emits one warped VRT per source band (see [Geospatial grid](#geospatial-grid)), and returns the assembled `xarray.Dataset`. `DATASET_NAMES` is the fixed set of seven raster datasets, asserted to equal every `RasterDataset` in the registry. -- `encode_transition_image(from_img, to_img) -> EEImage`: packs two category bands via `from_img.leftShift(4).bitwiseOr(to_img).uint8()`, with `0` written where `from == to`. -- `decode_from_image(encoded) -> EEImage`: `encoded.rightShift(4)`. -- `decode_to_image(encoded) -> EEImage`: `encoded.bitwiseAnd(0xF)`. +### Emissions (`emissions.py`) -#### asset_management.py +`python jdluc/emissions.py `. `workflow(tile_ids)` (`@gcs.cache(version=2)`) calls `harmonize.workflow` then: -Reusable helpers for working with GEE asset IDs: `list_assets_matching(prefix)`, `parse_version_from_asset_id(asset_id)`, `delete_asset_safely(asset_id)`, and related utilities. +1. **Land classes** — `get_land_class` maps each year's GLAD band into the 7-member `LandClass` enum (disjoint value ranges summed without collision). +2. **Vegetation carbon & emissions** — above-ground carbon (Harris AGB × CF), below-ground carbon (Huang BGB with R:S fallback), dead organic matter (climate-zone factors), and grassland carbon; differenced across consecutive epochs into `vegetation-emissions:{b}-{a}`. +3. **Soil emissions** — IPCC stock-change `F_LU` deltas keyed on land class + climate zone, plus peatland transformation, into `soil-emissions:{b}-{a}`. +4. **Peatland occupation** — annual land-management emissions on cultivated peatland pixels. +5. **Allocation & area** — sum vegetation + soil per epoch, apply the GHGP linear-discount weights (`EPOCH_TO_LINEAR_DISCOUNT_WEIGHT`), add peatland occupation, and scale by `get_hectares_per_pixel`. -#### version.py +Returns the [per-pixel zarr dataset](#per-pixel-emissions-zarr). -Exposes two stage-scoped version functions whose outputs are embedded in the cached asset names for each stage: +### Attribution (`attribution.py`) -- `compute_transform_version()` — SHA derived from the git state of `transform/` plus `utils/` -- `compute_publish_version()` — SHA derived from the git state of `publish/` plus `utils/` +`python jdluc/attribution.py --admin-id USA008 [--skip-glad-crop-filter]` (`--admin-id` repeatable; defaults to Delaware `USA008`). `workflow(admin_ids, crops, skip_glad_crop_filter)` (`@gcs.cache(version=1)`) returns a `pandas.DataFrame` indexed by `(admin_level, crop_name, jurisdiction_name)`. -Extract-stage assets are versioned by their upstream dataset version only (see the Extract section) and do not call into this module. +For each (admin id, crop): the per-pixel emissions are clipped to the jurisdiction polygon from the matching World Bank `VectorDataset` (`rio.clip`, `drop=False`), masked to the crop's CDL `CropClass` codes (corn = 1, soy = 5, wheat = 22/23/24 via the `Crop` enum), and — unless `--skip-glad-crop-filter` — further restricted to pixels where `land-class:2020 == CROPLAND`. The masked emissions and areas are summed into a `JurisdictionalCropEmission`. World Bank admin levels (`AdminLevel`: `NATIONAL`=0, `PROVINCIAL`=1, `DISTRICT`=2) replace TIGER county FIPS; US states are the provincial level (`USA001`…`USA051`). -For each function, the resulting version string has two forms: +### Emissions factors (`emissions_factors.py`) -- **Clean** (working tree matches HEAD for the hashed files): `{HEAD SHA[:12]}` (e.g., `a0d76ac0aa12`) -- **Dirty** (working tree differs): `{HEAD SHA[:12]}-dirty-{sha256(diff)[:8]}` (e.g., `a0d76ac0aa12-dirty-3f2b8c01`) +`python jdluc/emissions_factors.py --admin-id USA008`. `workflow(admin_ids, crops, skip_glad_crop_filter)` (`@gcs.cache(version=0)`) takes the `attribution.workflow` rollup, joins it to NASS yields — loaded by `load_yields()` from the ingested `USDA_NASS_QUICKSTATS` parquet — on `(admin_id, crop_name)` using the 4-year (`NASS_YIELD_YEARS` = 2017–2020) mean yield, and computes `total_production_kg`, `emissions_factor_kgco2e_per_kg`, and `peatland_occupation_fraction`. Returns the [EF table](#per-jurisdiction-crop-emissions-factors-parquet). -## CLI +### Yields (`datasets/usda_nass_quickstats.py`) -The `cli.py` entrypoint runs the full pipeline for a named canned region (e.g., `--region=delaware`) +Yields are not a standalone stage — they are ingested like any other source. The `USDA_NASS_QUICKSTATS` `TabularDataset`'s `get_records_for_tile_id` pulls state-level YIELD records (corn, soybeans, wheat) from the NASS QuickStats API (`https://quickstats.nass.usda.gov/api/api_GET/`, key from `.env`), converts bu/acre → kg/ha using 7 CFR 810 bushel weights and `HA_PER_ACRE = 0.40468564`, and keys results by World Bank admin id via `STATE_FIPS_TO_ADMIN_ID`. The ingested parquet is read back by `emissions_factors.load_yields()`. ## Testing -**Unit tests** validate pure logic — category mappings, version string formatting, yield calculations, transition encoding/decoding — with no external dependencies. Fast, runnable in CI without credentials. - -**Integration tests** run against live GEE on a small test region (Delaware) and assert that outputs have the expected bands, non-zero values, and plausible magnitudes. A session-scoped `conftest.py` fixture handles `initialize_gee` once per run. Integration tests are marked `@pytest.mark.integration` and excluded from CI by default (`pytest -m "not integration"`); they run on demand locally or in manual pre-merge validation. - -### Selected basic invariants - -Some simple invariants tested by the integration suite should include: - -- **Non-negativity**: all emissions and area values are ≥ 0 at both pixel level and aggregated-row level. Zero-valued non-emissive transitions are exactly zero, not small negatives. -- **Allocated ≤ total**: for every row, `allocated_emissions_2020_tco2 ≤ total_emissions_tco2`. Follows logically from the GHGP 20-year lookback being a weighted subset of total epoch emissions. -- **Components sum to total**: per-category emissions components (forest + short-vegetation + SOC + peatland) sum to the total reported per row, within floating-point tolerance. -- **Epoch sensitivity**: the four per-epoch transition bands are not all equal — the pipeline is actually sensitive to when a transition occurred within the analysis window. +Tests live in `jdluc/__tests__/` (`test_utils.py`, `test_datasets.py`, `test_harmonize.py`, `test_emissions.py`). Unit tests cover pure logic (land-class mapping, grid math, chunking, transition/epoch weighting) with no network. ## Reproducibility -An additional benefit of the asset caching strategy described throughout the pipeline above is that it should usually be possible to recover the full history that produced any pipeline output. This is not a perfect guarantee, but it should be possible to at least get very close, most of the time. - -### Input data - -Depends on whether the input is GEE-native: - -- **Non-GEE-native inputs** (Harris AGB, Huang BGB, NASS QuickStats, IPCC climate zones, GFW Peatlands): the original source files are archived in the GCS extract mirror (see "Source-data mirroring" above), regardless of whether the upstream publisher still hosts them. -- **GEE-native inputs** (GLAD GLCLUC, CDL, SoilGrids, TIGER): we do not archive a Watershed-side copy of these assets, but upstream stability is high. +Two mechanisms make outputs recoverable: -### Code recovery - -- **Extract-stage outputs** carry an upstream-dataset-version suffix (e.g. `harris_agb_conus_v2021`). File creation date can be cross-referenced to commit history if the logic to generate these assets from the archived GCS files appears to have changed relative to the current HEAD. But extract logic should be slow changing so this should be rare. -- **Transform and Publish outputs** carry a commit SHA, provided the working tree was clean when the code was run. This allows recovery of the exact committed code. A `-dirty` suffix means the exact working-tree state is not recoverable from git alone, but it's at least possible to recover the last pre-run commit. +- **Ingested sources** are mirrored in the ingest bucket under versioned, provenance-tagged prefixes, so a given source version is re-readable even if the upstream publisher changes. +- **Stage outputs** are content-addressed by `gcs.cache` (module + qualname + version + args), so an output's cache key identifies the exact code path and inputs that produced it. Bumping a stage `version` or any input forces recomputation. ## Scale -The cost and speed results below are empirically measured outcomes for the current implementation at the "Limited" GEE subscription tier (the lowest cost tier, which therefore gets lowest priority on GEE's shared resource pool). We've benchmarked two single-state runs of differing sizes (Delaware, Iowa), one regional run (Great Plains = IA + NE + SD), and one full CONUS run. - -Further pipeline optimization is likely possible. - -### Cost - -Compute and cost by stage for the four benchmark regions. This reflects `transform/` steps only — we consider `extract/` to be a well-amortized one-time cost, such that the dollar cost becomes fairly trivial, and the BigQuery `publish/` exports are also negligible (~0 EECU-hr / <1 min wall-clock for both tables on CONUS). Costs are stated at GEE's list rate of $.40 per EECU. - - -| Stage | DE EECU-hr | DE $ | IA EECU-hr | IA $ | Great Plains EECU-hr | Great Plains $ | CONUS EECU-hr | CONUS $ | -| ----------- | ---------- | --------- | ---------- | --------- | ----------- | ---------- | ------------- | ----------- | -| land_use | 0.030 | $0.012 | 0.70 | $0.28 | 2.49 | $1.00 | 46.14 | $18.46 | -| emissions | 0.129 | $0.052 | 1.78 | $0.71 | 8.03 | $3.21 | 127.64 | $51.06 | -| transitions | 0.108 | $0.043 | 1.78 | $0.71 | 7.98 | $3.19 | 294.49 | $117.79 | -| crops | 0.158 | $0.063 | 2.67 | $1.07 | 12.52 | $5.01 | 184.43 | $73.77 | -| **Total** | **0.42** | **$0.17** | **6.92** | **$2.77** | **31.02** | **$12.41** | **652.70** | **$261.08** | - - -Region size and per-pixel cost, calculated two ways: per **bounding-box pixel** (what the pipeline actually streams over, including over-water area that the FIPS mask later drops cheaply), and per **polygon pixel** (real on-shore land area at 30 m): - - -| Region | Bbox px | Polygon px | nEECU-hr / bbox px | nEECU-hr / polygon px | -| --------------- | ------- | ---------- | ------------------ | --------------------- | -| DE | 17.9 M | 7 M | 24 | 60 | -| IA | 250 M | 161 M | 28 | 43 | -| Great Plains | 900 M | 600 M | 35 | 52 | -| CONUS | 23 B | 8.4 B | 28 | 78 | - - -The smaller-region (DE → GP3) trend looked like a pretty steady ^1.10 power law in bbox pixels. CONUS broke that: per-bbox-pixel cost came in *lower* than GP3 (28 vs 35), even as per-polygon-pixel cost rose meaningfully (52 → 78). Both seem to reflect the same underlying fact — the CONUS bounding box is a much worse fit for the polygon than our 3 state Great Plains test region (bbox/polygon ratio 2.7× vs 1.5×). This chunk of the CONUS bbox sits over the Atlantic, Pacific, and Gulf and gets masked out by the within-pipeline `fips_mask` cheaply. Overall, cost-per-area is generally climbing modestly with region, with bbox pixels providing the slightly more reliable, but still imperfect, scaling predictor. - -#### Global extrapolation - -CONUS land area is ~7.6 M km² vs ~134 M km² of global ice-free land — a ~17.6× pixel-count ratio at a uniform 30 m grid (≈148 B polygon pixels). Bbox is harder to calculate, but let's assume we can maintain bbox/polygon ratios across whatever shards we decide to use for a global run. The U.S. is not particularly favorable in this regard, so that doesn't seem unduly optimistic. - -Anchoring on the CONUS actual (652.7 EECU-hr / $261.08) and assuming roughly linear scaling, similar to the GP3 -> CONUS trend: $4,600. The power-law (^1.10) scale from CONUS is not much worse: 17.6^1.10 ≈ 23.5× → $6,100. - -So plausible range **$4,500–$6,000** for a single global run. - -### Speed - -#### Extract - -The extract stage is limited by GEE's per-project ingest concurrency cap, which is maxed out by the tile-based datasets (Harris AGB and the GFW Peatlands raster). Our concurrency cap is currently about ~20 simultaneous tasks on the Limited tier. With this concurrency, the tile-based datasets finish in ~30 minutes for full CONUS extract (which the pipeline performs even for regional runs). The single-asset extracts (Huang BGB, NASS, IPCC, county_fips) take 1-30 minutes each. The county_fips paint adds ~30 minutes one-time at first run and is then cache-hit forever. Total clean-cache extract stage is on the order of two hours for all assets. - -#### Transform/Publish - -The transform stage is effectively O(1), subject to GEE's willingness to allocate resources. - -**Per-stage GEE wall-clock** - - -| Stage | Delaware | Iowa | Great Plains | CONUS | -| ----------------------------------- | ---------- | ----------- | ------------ | --------------------- | -| land_use | 4.2 min | 23.9 min | 17.8 min | 130.6 min | -| emissions | 3.7 min | 11.7 min | 24.4 min | 144.3 min | -| transitions (parallel w/crops) | 0.9 min | 4.1 min | 3.6 min | 66.3 min | -| crops (parallel w/transitions) | 0.9 min | 3.1 min | 2.6 min | 22.5 min | -| **GEE-side pipeline total** | **~9 min** | **~40 min** | **~46 min** | **~341 min (5.7 hr)** | - -In our smaller-region runs, the Great Plains land_use stage actually ran faster than IA's despite 3× more pixels. But that parallelism saturated for us between the Great Plains and CONUS scale: CONUS was \~20× EECU and \~7× wall-clock vs. Great Plains. It's unclear whether this would work out the same on a second trial, or whether it's subject to the priority of other jobs in the GEE queue at runtime. - -At the Limited tier, we also observed queue waits of 5-15 min per task, which were very significant for the smaller regions, but a smaller fraction of total wall-clock as exports get bigger. +The pipeline runs on a single dask-capable host — no distributed cluster. The benchmark below is an **M4 MacBook (48 GB RAM)** with `NUMBER_OF_DASK_WORKERS=12` (threaded scheduler, `scheduler="threads"`), ingest `--concurrency=4`, and the ~4 GiB chunk target from `geo.get_chunk_size` (`max_bytes_per_chunk = 1<<32`). Figures are for the **CONUS** tile set, the largest of the three run (`DELAWARE` and `BAY_AREA` were also run for iteration but are far smaller/faster and not separately benchmarked). -#### Global extrapolation +| Stage | Wall-clock (CONUS) | Cached output | Output size | +|---|---|---|---| +| Ingest | ~30 m | source tiles (GeoTIFF) in ingest bucket | 36.4 GiB | +| Harmonize | ~40 m | common-grid `xarray.Dataset` (zarr) | 1.3 TiB | +| Emissions | ~2 h | per-pixel emissions (zarr) | 3.7 TiB | +| Attribution | ~1 h | rollup (parquet) | < 1 MiB | +| Emissions factors | ~5 s | EF table (parquet) | < 1 MiB | -With job-level parallelism saturating at approximately Great Plains scale, the CONUS → Global jump (\~21× pixels) likely lands in the 10× wall-clock range, even assuming we reintroduce some parallelism on our side by splitting the global run into multiple jobs. This comes out to ~2-3 days of run time. A higher service tier could potentially bring this back inside a day. +End-to-end ≈ **5 h** for CONUS. **Harmonize, emissions, and attribution are the cost centers**. Storage is dominated by the two zarr stores — the harmonized dataset (1.3 TiB) and the per-pixel emissions (3.7 TiB); attribution is compute-heavy despite its kilobyte-scale parquet output, because it streams the full per-pixel zarr through the clip/mask/rollup, and the final emissions-factors join is effectively instantaneous. The 48 GB host comfortably absorbed 12 threaded workers at the chunk target. diff --git a/uv.lock b/uv.lock index 3a96d04..7620fc4 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,9 @@ revision = 3 requires-python = "==3.14.*" resolution-markers = [ "platform_machine == 'ARM64' and sys_platform == 'win32'", - "platform_machine != 'ARM64' or sys_platform != 'win32'", + "platform_machine != 'ARM64' and sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -16,72 +18,149 @@ wheels = [ ] [[package]] -name = "anywidget" -version = "0.11.0" +name = "aiohappyeyeballs" +version = "2.6.2" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipywidgets" }, - { name = "psygnal" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/31/0491d707c674b34267f55d96d6a7148e55e7b6718a271686232cf295fbe2/anywidget-0.11.0.tar.gz", hash = "sha256:6695fbef9449cf8c27f421b96c5837aa37f909ec1f60cfa33add333e1b70b169", size = 426999, upload-time = "2026-04-27T23:42:09.576Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591, upload-time = "2026-05-20T15:12:24.631Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/c2/8fec8e8e2eb920cc2280f569144080cd58622a2eda83bfa4c0c354a63264/anywidget-0.11.0-py3-none-any.whl", hash = "sha256:c574d9acc6503ad27b37a9acea48f957a8ba7c9c9876cfcb37898931c098ce9d", size = 317341, upload-time = "2026-04-27T23:42:08.356Z" }, + { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062, upload-time = "2026-05-20T15:12:23.328Z" }, ] [[package]] -name = "asttokens" -version = "3.0.1" +name = "aiohttp" +version = "3.14.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, + { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, + { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, + { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, + { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, + { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, + { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, + { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, + { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, + { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, + { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, + { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, + { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, + { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +dependencies = [ + { name = "frozenlist" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] [[package]] -name = "bqplot" -version = "0.12.46" +name = "annotated-types" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipywidgets" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "traitlets" }, - { name = "traittypes" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/63/e63f6cec5f08aaecfd6420f84a261050d168ffe1a887fe82e4b1f8c35b66/bqplot-0.12.46.tar.gz", hash = "sha256:94174be3eb241c9fa1d697a54313289a0bd3fe880add96c6640d3163bdd3d22a", size = 1205847, upload-time = "2026-04-28T10:21:19.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/49/a8d2bc745fe58f4db87202564379d10304f1fc3dff79a89a759c6fb9dbb2/bqplot-0.12.46-py2.py3-none-any.whl", hash = "sha256:39301c4fa92a85188ba1e50fc4d8977fa132ac562916b3427cdbd9e853337dcd", size = 1237522, upload-time = "2026-04-28T10:21:17.535Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "ast-serialize" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/9d/09e27731bd5864a9ce04e3244074e674bb8936bf62b45e0357248717adac/ast_serialize-0.5.0.tar.gz", hash = "sha256:5880091bfe6f4f986f22866375c2e884843e7a0b6343ae41aeea659613d879b6", size = 61157, upload-time = "2026-05-17T17:48:29.429Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/9a/13dde51ba9e15f8b97957ab7cb0120d0e381524d651c6bd630b9c359227f/ast_serialize-0.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8f5c14f169eb0972c0c21bada5358b23d6047c76583b005234f865b11f1fa00a", size = 1183520, upload-time = "2026-05-17T17:47:30.831Z" }, + { url = "https://files.pythonhosted.org/packages/37/de/5a7f0a9fe68944f536632a5af84676739c7d2582be42deb082634bf3a754/ast_serialize-0.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7d1a2de9de5be04652f0ed60738356ef94f66db37924a9499fffe98dc491aa0b", size = 1175779, upload-time = "2026-05-17T17:47:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/9c/81/0bb853e76e4f6e9a1855d569003c59e19ffac45f7079d91505d1bb212f92/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be5173fb66f9b49026d9d5a2ff0fc7c7009077107c0eb285b2d60fdf1fe10bd1", size = 1233750, upload-time = "2026-05-17T17:47:34.731Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d3/4cf705beeccc08754d0bbda99aefff26110e209b9a07ac8a6b60eec48531/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f8015cd071ac1339924ee2b8098c93e00e155f30a16f40ec9816fcf84f4753f6", size = 1235942, upload-time = "2026-05-17T17:47:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/26/c8/ee097e437ea27dd2b8b227865c875492b585650a5802a22d82b304c8201b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5499e8797edff2a9186aa313ed382c6b422e798e9332d9953badcee6e69a88f2", size = 1442517, upload-time = "2026-05-17T17:47:38.17Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bd/68063442838f1ba68ec72b5436430bc75b3bb17a1a3c3063f09b0c05ae2b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6848f2a093fb5548751a9a09bff8fcd229e2bbeb0e3331f391b6ae6d26cd9903", size = 1254081, upload-time = "2026-05-17T17:47:39.826Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/1e520793bc6a4e4524a6ab022391e827825eaa0c3811828bfdc6852eca26/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:832d4c998e0b091fd60a6d6bceee535483c4d490de9ba85003af835225719261", size = 1259910, upload-time = "2026-05-17T17:47:41.369Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e1/49b60f467979979cfe6913b43948ff25bca971ad0591d181812f163a988e/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:16db7c62ec0b8efe1d7afd283a388d8f74f2605d56032e5a37747d2de8dba027", size = 1250678, upload-time = "2026-05-17T17:47:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/74/ba/66ab9555de6275677566f6574e5ef6c29cb185ea866f643bc06f8280a8ee/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf5eb061eb5bccade4128ad42da33787d72f6013809cd1b590376ece8b3c937", size = 1301603, upload-time = "2026-05-17T17:47:46.256Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/6aca9b9abc710014b2be9059689e5dd1679339e78f567ffb4d255a9e2050/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:104e4a35bd7c124173c41760ef9aaea17ddb3f86c65cb643671d59afbe3ee94c", size = 1410332, upload-time = "2026-05-17T17:47:47.899Z" }, + { url = "https://files.pythonhosted.org/packages/47/68/2f76594432a22581ecf878b5e75a9b8601c24b2241cf0bbeb1e21fcf370c/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:36be371028fc1675acb38a331bde160dbab7ff907fdf00b67eb6911aa106951b", size = 1509979, upload-time = "2026-05-17T17:47:50.942Z" }, + { url = "https://files.pythonhosted.org/packages/40/ac/a93c9b58292653f6c595752f677a08e608f903b710594909e9231a389b3b/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:061ee58bdb52341c8201a6df41182a977736bae3b7ded87ca7176ca25a8a47ab", size = 1505002, upload-time = "2026-05-17T17:47:54.093Z" }, + { url = "https://files.pythonhosted.org/packages/14/2e/b278f68c497ee2f1d1576cbbef8db5281cd4a5f2db040537592ac9c8862e/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b15219e9cdc9f53f6f4cb51c009203507228226148c05c5e8fe451c28b435eb3", size = 1456231, upload-time = "2026-05-17T17:47:56.311Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/419be1c566a4c504cd8fd60ce2f84e790f295495c0f327cfaeadf3d51012/ast_serialize-0.5.0-cp314-cp314t-win32.whl", hash = "sha256:842d1c004bb466c7df036f95fabef789570541922b10976b12f5592a69cf0b38", size = 1058668, upload-time = "2026-05-17T17:47:58.305Z" }, + { url = "https://files.pythonhosted.org/packages/03/6f/c9d4d549295ed05111aeb8853232d1afd9d0a179fddb01eeffbb3a4a6842/ast_serialize-0.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b0c06d760909b095cc466356dfccd05a1c7233a6ca191c020dca2c6a6f16c24c", size = 1101075, upload-time = "2026-05-17T17:48:00.35Z" }, + { url = "https://files.pythonhosted.org/packages/d0/8e/d00c5ab30c58222e07d62956fca86c59d91b9ad32997e633c38b526623a3/ast_serialize-0.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:787baedb0262cc49e8ce37cc15c00ae818e46a165a3b36f5e21ed174998104cb", size = 1075347, upload-time = "2026-05-17T17:48:01.753Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9e/dc2530acb3a60dc6e46d65abf27d1d9f86721694757906a148d90a6860de/ast_serialize-0.5.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0668aa9459cfa8c9c49ddd2163ebcf43088ba045ef7492af6fe22e0098303101", size = 1191380, upload-time = "2026-05-17T17:48:03.738Z" }, + { url = "https://files.pythonhosted.org/packages/26/0a/bd3d18a582f273d6c843d16bb9e22e9e16365ff7991e92f18f798e9f1224/ast_serialize-0.5.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bf683d6363edf2b39eed6b6d4fe22d34b6203867a67e27134d9e2a2680c4bc4a", size = 1183879, upload-time = "2026-05-17T17:48:05.463Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/1f919100f8620887af58fcc381c61a1f218cdf89c6e155f87b213e61010a/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc22cf0c9be65e71cf88fda130af60d61eb4a79370ad4cfe7900d48a4aa2211", size = 1244529, upload-time = "2026-05-17T17:48:07.008Z" }, + { url = "https://files.pythonhosted.org/packages/c6/ca/6376559dcce707cdbc1d0d9a13c8d3baaaa501e949ce0ebdc4230cd881aa/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f66173891548c9f2726bf27957b41cabce12fa679dc6da505ddbde4d4b3b31cf", size = 1240560, upload-time = "2026-05-17T17:48:08.46Z" }, + { url = "https://files.pythonhosted.org/packages/35/b2/a620e206b5aeb7efbf2710336df57d457cffbb3991076bbcc1147ef9abd4/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e42d729ef2be96a14efbad355093284739e3670ece3e534f82cc8832790911d9", size = 1451172, upload-time = "2026-05-17T17:48:09.922Z" }, + { url = "https://files.pythonhosted.org/packages/fa/e0/4ad5c04c24a40481b2935ce9a0ccdb6023dc8b667167d06ae530cc3512f2/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b725026bafa801dbd7310eb13a75f0a2e370e7e51b2cb225f9d21fcfadf919ee", size = 1265072, upload-time = "2026-05-17T17:48:11.469Z" }, + { url = "https://files.pythonhosted.org/packages/b2/71/4d1d479aa56d0101c40e17720c3d6ac2af7269ea0487a80b18e7bfd1a5b7/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b54f60c1d78767a53b67eaa663f0dfac3afe606aa07f1301572f588b73d64809", size = 1270488, upload-time = "2026-05-17T17:48:13.575Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/0de1bbe06f6edef9fde4ed12ca8e7b3ec7e6e2bd4e672c5af487f7957665/ast_serialize-0.5.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:27d51654fc240a1e87e742d353d98eb45b75f62f129086b3596ab53df2ac2a43", size = 1260702, upload-time = "2026-05-17T17:48:15.141Z" }, + { url = "https://files.pythonhosted.org/packages/75/61/e00872439cfdddcc3c1b6cdaa6e5d904ba8e26a18807c67c4e14409d0ca8/ast_serialize-0.5.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c36237c46dd1674542f2109740ea5ea485a169bf1431939ada0434e17934", size = 1311182, upload-time = "2026-05-17T17:48:16.779Z" }, + { url = "https://files.pythonhosted.org/packages/76/8e/699a5b955f7926956c95e9e1d74132acad73c2fe7a426f94da89123c20aa/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1943db345233cc7194a470f13afa9c59772c0b123dea0c9414c4d4ca54369759", size = 1421410, upload-time = "2026-05-17T17:48:18.527Z" }, + { url = "https://files.pythonhosted.org/packages/a9/ae/d5b7626874478997adc7a29ab28accf21e596fb590c944290401dfd0b29e/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df1c00022cbbcb064bfaa505aa9c9295362443ce5dacb459d1331d3da353f887", size = 1516587, upload-time = "2026-05-17T17:48:20.133Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ce/b59e02a82d9c4244d64cde502e0b00e83e38816abe19155ceb5437402c7f/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:cae65289fc456fde04af979a2be09302ef5d8ab92ef23e596d6746dc267ada27", size = 1515171, upload-time = "2026-05-17T17:48:21.921Z" }, + { url = "https://files.pythonhosted.org/packages/8b/38/d8d90042747d05aa08d4efcf1c99035a5f670a6bf4c214d31644392afbca/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:239a4c354e8d676e9d94631d1d4a64edc6b266f86ff3a5a80aedd344f342c01d", size = 1464668, upload-time = "2026-05-17T17:48:23.544Z" }, + { url = "https://files.pythonhosted.org/packages/dd/51/5b840c4df7334104cecffa28f23904fe81ca89ca223d2450e288de39fd3c/ast_serialize-0.5.0-cp39-abi3-win32.whl", hash = "sha256:143a4ef63285a075871908fda3672dc21864b83a8ec3ee12304aa3e4c5387b9a", size = 1068311, upload-time = "2026-05-17T17:48:25.027Z" }, + { url = "https://files.pythonhosted.org/packages/41/11/ca5672c7d491825bc4cd6702dea106a6b60d928707712ec257c7833ae476/ast_serialize-0.5.0-cp39-abi3-win_amd64.whl", hash = "sha256:cf25572c526add400f26a4750dc6ce0c3bb93fc1f75e7ae0cad4ce4f2cd5c590", size = 1108931, upload-time = "2026-05-17T17:48:26.591Z" }, + { url = "https://files.pythonhosted.org/packages/45/19/cc8bd127d28a43da249aa955cfd164cf8fd534e79e42cea96c4854d72fd0/ast_serialize-0.5.0-cp39-abi3-win_arm64.whl", hash = "sha256:92a31c9c20d25a076edaeec76b128a3535d74a24f340b9a8a7e96c9b86dc9642", size = 1081181, upload-time = "2026-05-17T17:48:28.122Z" }, ] [[package]] -name = "branca" -version = "0.8.2" +name = "attrs" +version = "26.1.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/14/9d409124bda3f4ab7af3802aba07181d1fd56aa96cc4b999faea6a27a0d2/branca-0.8.2.tar.gz", hash = "sha256:e5040f4c286e973658c27de9225c1a5a7356dd0702a7c8d84c0f0dfbde388fe7", size = 27890, upload-time = "2025-10-06T10:28:20.305Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/50/fc9680058e63161f2f63165b84c957a0df1415431104c408e8104a3a18ef/branca-0.8.2-py3-none-any.whl", hash = "sha256:2ebaef3983e3312733c1ae2b793b0a8ba3e1c4edeb7598e10328505280cf2f7c", size = 26193, upload-time = "2025-10-06T10:28:19.255Z" }, + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "certifi" -version = "2026.4.22" +version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, ] [[package]] @@ -193,26 +272,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, -] - -[[package]] -name = "click-plugins" -version = "1.1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] @@ -228,281 +295,243 @@ wheels = [ ] [[package]] -name = "colorama" -version = "0.4.6" +name = "cloudpickle" +version = "3.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, ] [[package]] -name = "comm" -version = "0.2.3" +name = "colorama" +version = "0.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, -] - -[[package]] -name = "contourpy" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/8b/4546f3ab60f78c514ffb7d01a0bd743f90de36f0019d1be84d0a708a580a/contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a", size = 292189, upload-time = "2025-07-26T12:02:16.095Z" }, - { url = "https://files.pythonhosted.org/packages/fd/e1/3542a9cb596cadd76fcef413f19c79216e002623158befe6daa03dbfa88c/contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77", size = 273251, upload-time = "2025-07-26T12:02:17.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, - { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, - { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, - { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, - { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e2/366af18a6d386f41132a48f033cbd2102e9b0cf6345d35ff0826cd984566/contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d", size = 189692, upload-time = "2025-07-26T12:02:30.128Z" }, - { url = "https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd", size = 232424, upload-time = "2025-07-26T12:02:31.395Z" }, - { url = "https://files.pythonhosted.org/packages/18/79/a9416650df9b525737ab521aa181ccc42d56016d2123ddcb7b58e926a42c/contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339", size = 198300, upload-time = "2025-07-26T12:02:32.956Z" }, - { url = "https://files.pythonhosted.org/packages/1f/42/38c159a7d0f2b7b9c04c64ab317042bb6952b713ba875c1681529a2932fe/contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772", size = 306769, upload-time = "2025-07-26T12:02:34.2Z" }, - { url = "https://files.pythonhosted.org/packages/c3/6c/26a8205f24bca10974e77460de68d3d7c63e282e23782f1239f226fcae6f/contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77", size = 287892, upload-time = "2025-07-26T12:02:35.807Z" }, - { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, - { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, - { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, - { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, - { url = "https://files.pythonhosted.org/packages/93/8a/68a4ec5c55a2971213d29a9374913f7e9f18581945a7a31d1a39b5d2dfe5/contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae", size = 202428, upload-time = "2025-07-26T12:02:48.691Z" }, - { url = "https://files.pythonhosted.org/packages/fa/96/fd9f641ffedc4fa3ace923af73b9d07e869496c9cc7a459103e6e978992f/contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc", size = 250331, upload-time = "2025-07-26T12:02:50.137Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8c/469afb6465b853afff216f9528ffda78a915ff880ed58813ba4faf4ba0b6/contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b", size = 203831, upload-time = "2025-07-26T12:02:51.449Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "cryptography" -version = "47.0.0" +version = "49.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/98/40dfe932134bdcae4f6ab5927c87488754bf9eb79297d7e0070b78dd58e9/cryptography-47.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:160ad728f128972d362e714054f6ba0067cab7fb350c5202a9ae8ae4ce3ef1a0", size = 7912214, upload-time = "2026-04-24T19:53:03.864Z" }, - { url = "https://files.pythonhosted.org/packages/34/c6/2733531243fba725f58611b918056b277692f1033373dcc8bd01af1c05d4/cryptography-47.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b9a8943e359b7615db1a3ba587994618e094ff3d6fa5a390c73d079ce18b3973", size = 4644617, upload-time = "2026-04-24T19:53:06.909Z" }, - { url = "https://files.pythonhosted.org/packages/00/e3/b27be1a670a9b87f855d211cf0e1174a5d721216b7616bd52d8581d912ed/cryptography-47.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5c15764f261394b22aef6b00252f5195f46f2ca300bec57149474e2538b31f8", size = 4668186, upload-time = "2026-04-24T19:53:09.053Z" }, - { url = "https://files.pythonhosted.org/packages/81/b9/8443cfe5d17d482d348cee7048acf502bb89a51b6382f06240fd290d4ca3/cryptography-47.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9c59ab0e0fa3a180a5a9c59f3a5abe3ef90d474bc56d7fadfbe80359491b615b", size = 4651244, upload-time = "2026-04-24T19:53:11.217Z" }, - { url = "https://files.pythonhosted.org/packages/5d/5e/13ed0cdd0eb88ba159d6dd5ebfece8cb901dbcf1ae5ac4072e28b55d3153/cryptography-47.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:34b4358b925a5ea3e14384ca781a2c0ef7ac219b57bb9eacc4457078e2b19f92", size = 5252906, upload-time = "2026-04-24T19:53:13.532Z" }, - { url = "https://files.pythonhosted.org/packages/64/16/ed058e1df0f33d440217cd120d41d5dda9dd215a80b8187f68483185af82/cryptography-47.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0024b87d47ae2399165a6bfb20d24888881eeab83ae2566d62467c5ff0030ce7", size = 4701842, upload-time = "2026-04-24T19:53:15.618Z" }, - { url = "https://files.pythonhosted.org/packages/02/e0/3d30986b30fdbd9e969abbdf8ba00ed0618615144341faeb57f395a084fe/cryptography-47.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:1e47422b5557bb82d3fff997e8d92cff4e28b9789576984f08c248d2b3535d93", size = 4289313, upload-time = "2026-04-24T19:53:17.755Z" }, - { url = "https://files.pythonhosted.org/packages/df/fd/32db38e3ad0cb331f0691cb4c7a8a6f176f679124dee746b3af6633db4d9/cryptography-47.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:6f29f36582e6151d9686235e586dd35bb67491f024767d10b842e520dc6a07ac", size = 4650964, upload-time = "2026-04-24T19:53:20.062Z" }, - { url = "https://files.pythonhosted.org/packages/86/53/5395d944dfd48cb1f67917f533c609c34347185ef15eb4308024c876f274/cryptography-47.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a9b761f012a943b7de0e828843c5688d0de94a0578d44d6c85a1bae32f87791f", size = 5207817, upload-time = "2026-04-24T19:53:22.498Z" }, - { url = "https://files.pythonhosted.org/packages/34/4f/e5711b28e1901f7d480a2b1b688b645aa4c77c73f10731ed17e7f7db3f0d/cryptography-47.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4e1de79e047e25d6e9f8cea71c86b4a53aced64134f0f003bbcbf3655fd172c8", size = 4701544, upload-time = "2026-04-24T19:53:24.356Z" }, - { url = "https://files.pythonhosted.org/packages/22/22/c8ddc25de3010fc8da447648f5a092c40e7a8fadf01dd6d255d9c0b9373d/cryptography-47.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef6b3634087f18d2155b1e8ce264e5345a753da2c5fa9815e7d41315c90f8318", size = 4783536, upload-time = "2026-04-24T19:53:26.665Z" }, - { url = "https://files.pythonhosted.org/packages/66/b6/d4a68f4ea999c6d89e8498579cba1c5fcba4276284de7773b17e4fa69293/cryptography-47.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:11dbb9f50a0f1bb9757b3d8c27c1101780efb8f0bdecfb12439c22a74d64c001", size = 4926106, upload-time = "2026-04-24T19:53:28.686Z" }, - { url = "https://files.pythonhosted.org/packages/54/ed/5f524db1fade9c013aa618e1c99c6ed05e8ffc9ceee6cda22fed22dda3f4/cryptography-47.0.0-cp311-abi3-win32.whl", hash = "sha256:7fda2f02c9015db3f42bb8a22324a454516ed10a8c29ca6ece6cdbb5efe2a203", size = 3258581, upload-time = "2026-04-24T19:53:31.058Z" }, - { url = "https://files.pythonhosted.org/packages/b2/dc/1b901990b174786569029f67542b3edf72ac068b6c3c8683c17e6a2f5363/cryptography-47.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:f5c3296dab66202f1b18a91fa266be93d6aa0c2806ea3d67762c69f60adc71aa", size = 3775309, upload-time = "2026-04-24T19:53:33.054Z" }, - { url = "https://files.pythonhosted.org/packages/14/88/7aa18ad9c11bc87689affa5ce4368d884b517502d75739d475fc6f4a03c7/cryptography-47.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:be12cb6a204f77ed968bcefe68086eb061695b540a3dd05edac507a3111b25f0", size = 7904299, upload-time = "2026-04-24T19:53:35.003Z" }, - { url = "https://files.pythonhosted.org/packages/07/55/c18f75724544872f234678fdedc871391722cb34a2aee19faa9f63100bb2/cryptography-47.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2ebd84adf0728c039a3be2700289378e1c164afc6748df1a5ed456767bef9ba7", size = 4631180, upload-time = "2026-04-24T19:53:37.517Z" }, - { url = "https://files.pythonhosted.org/packages/ee/65/31a5cc0eaca99cec5bafffe155d407115d96136bb161e8b49e0ef73f09a7/cryptography-47.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f68d6fbc7fbbcfb0939fea72c3b96a9f9a6edfc0e1b1d29778a2066030418b1", size = 4653529, upload-time = "2026-04-24T19:53:39.775Z" }, - { url = "https://files.pythonhosted.org/packages/e5/bc/641c0519a495f3bfd0421b48d7cd325c4336578523ccd76ea322b6c29c7a/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:6651d32eff255423503aa276739da98c30f26c40cbeffcc6048e0d54ef704c0c", size = 4638570, upload-time = "2026-04-24T19:53:42.129Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f2/300327b0a47f6dc94dd8b71b57052aefe178bb51745073d73d80604f11ab/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3fb8fa48075fad7193f2e5496135c6a76ac4b2aa5a38433df0a539296b377829", size = 5238019, upload-time = "2026-04-24T19:53:44.577Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5a/5b5cf994391d4bf9d9c7efd4c66aabe4d95227256627f8fea6cff7dfadbd/cryptography-47.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11438c7518132d95f354fa01a4aa2f806d172a061a7bed18cf18cbdacdb204d7", size = 4686832, upload-time = "2026-04-24T19:53:47.015Z" }, - { url = "https://files.pythonhosted.org/packages/dc/2c/ae950e28fd6475c852fc21a44db3e6b5bcc1261d1e370f2b6e42fa800fef/cryptography-47.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8c1a736bbb3288005796c3f7ccb9453360d7fed483b13b9f468aea5171432923", size = 4269301, upload-time = "2026-04-24T19:53:48.97Z" }, - { url = "https://files.pythonhosted.org/packages/67/fb/6a39782e150ffe5cc1b0018cb6ddc48bf7ca62b498d7539ffc8a758e977d/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:f1557695e5c2b86e204f6ce9470497848634100787935ab7adc5397c54abd7ab", size = 4638110, upload-time = "2026-04-24T19:53:51.011Z" }, - { url = "https://files.pythonhosted.org/packages/8e/d7/0b3c71090a76e5c203164a47688b697635ece006dcd2499ab3a4dbd3f0bd/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:f9a034b642b960767fb343766ae5ba6ad653f2e890ddd82955aef288ffea8736", size = 5194988, upload-time = "2026-04-24T19:53:52.962Z" }, - { url = "https://files.pythonhosted.org/packages/63/33/63a961498a9df51721ab578c5a2622661411fc520e00bd83b0cc64eb20c4/cryptography-47.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b1c76fca783aa7698eb21eb14f9c4aa09452248ee54a627d125025a43f83e7a7", size = 4686563, upload-time = "2026-04-24T19:53:55.274Z" }, - { url = "https://files.pythonhosted.org/packages/b7/bf/5ee5b145248f92250de86145d1c1d6edebbd57a7fe7caa4dedb5d4cf06a1/cryptography-47.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4f7722c97826770bab8ae92959a2e7b20a5e9e9bf4deae68fd86c3ca457bab52", size = 4770094, upload-time = "2026-04-24T19:53:57.753Z" }, - { url = "https://files.pythonhosted.org/packages/92/43/21d220b2da5d517773894dacdcdb5c682c28d3fffce65548cb06e87d5501/cryptography-47.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:09f6d7bf6724f8db8b32f11eccf23efc8e759924bc5603800335cf8859a3ddbd", size = 4913811, upload-time = "2026-04-24T19:54:00.236Z" }, - { url = "https://files.pythonhosted.org/packages/31/98/dc4ad376ac5f1a1a7d4a83f7b0c6f2bcad36b5d2d8f30aeb482d3a7d9582/cryptography-47.0.0-cp314-cp314t-win32.whl", hash = "sha256:6eebcaf0df1d21ce1f90605c9b432dd2c4f4ab665ac29a40d5e3fc68f51b5e63", size = 3237158, upload-time = "2026-04-24T19:54:02.606Z" }, - { url = "https://files.pythonhosted.org/packages/bc/da/97f62d18306b5133468bc3f8cc73a3111e8cdc8cf8d3e69474d6e5fd2d1b/cryptography-47.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:51c9313e90bd1690ec5a75ed047c27c0b8e6c570029712943d6116ef9a90620b", size = 3758706, upload-time = "2026-04-24T19:54:04.433Z" }, - { url = "https://files.pythonhosted.org/packages/e0/34/a4fae8ae7c3bc227460c9ae43f56abf1b911da0ec29e0ebac53bb0a4b6b7/cryptography-47.0.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:14432c8a9bcb37009784f9594a62fae211a2ae9543e96c92b2a8e4c3cd5cd0c4", size = 7904072, upload-time = "2026-04-24T19:54:06.411Z" }, - { url = "https://files.pythonhosted.org/packages/01/64/d7b1e54fdb69f22d24a64bb3e88dc718b31c7fb10ef0b9691a3cf7eeea6e/cryptography-47.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:07efe86201817e7d3c18781ca9770bc0db04e1e48c994be384e4602bc38f8f27", size = 4635767, upload-time = "2026-04-24T19:54:08.519Z" }, - { url = "https://files.pythonhosted.org/packages/8b/7b/cca826391fb2a94efdcdfe4631eb69306ee1cff0b22f664a412c90713877/cryptography-47.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b45761c6ec22b7c726d6a829558777e32d0f1c8be7c3f3480f9c912d5ee8a10", size = 4654350, upload-time = "2026-04-24T19:54:10.795Z" }, - { url = "https://files.pythonhosted.org/packages/4c/65/4b57bcc823f42a991627c51c2f68c9fd6eb1393c1756aac876cba2accae2/cryptography-47.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:edd4da498015da5b9f26d38d3bfc2e90257bfa9cbed1f6767c282a0025ae649b", size = 4643394, upload-time = "2026-04-24T19:54:13.275Z" }, - { url = "https://files.pythonhosted.org/packages/f4/c4/2c5fbeea70adbbca2bbae865e1d605d6a4a7f8dbd9d33eaf69645087f06c/cryptography-47.0.0-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9af828c0d5a65c70ec729cd7495a4bf1a67ecb66417b8f02ff125ab8a6326a74", size = 5225777, upload-time = "2026-04-24T19:54:15.18Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b8/ac57107ef32749d2b244e36069bb688792a363aaaa3acc9e3cf84c130315/cryptography-47.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:256d07c78a04d6b276f5df935a9923275f53bd1522f214447fdf365494e2d515", size = 4688771, upload-time = "2026-04-24T19:54:17.835Z" }, - { url = "https://files.pythonhosted.org/packages/56/fc/9f1de22ff8be99d991f240a46863c52d475404c408886c5a38d2b5c3bb26/cryptography-47.0.0-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:5d0e362ff51041b0c0d219cc7d6924d7b8996f57ce5712bdcef71eb3c65a59cc", size = 4270753, upload-time = "2026-04-24T19:54:19.963Z" }, - { url = "https://files.pythonhosted.org/packages/00/68/d70c852797aa68e8e48d12e5a87170c43f67bb4a59403627259dd57d15de/cryptography-47.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:1581aef4219f7ca2849d0250edaa3866212fb74bf5667284f46aa92f9e65c1ca", size = 4642911, upload-time = "2026-04-24T19:54:21.818Z" }, - { url = "https://files.pythonhosted.org/packages/a5/51/661cbee74f594c5d97ff82d34f10d5551c085ca4668645f4606ebd22bd5d/cryptography-47.0.0-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a49a3eb5341b9503fa3000a9a0db033161db90d47285291f53c2a9d2cd1b7f76", size = 5181411, upload-time = "2026-04-24T19:54:24.376Z" }, - { url = "https://files.pythonhosted.org/packages/94/87/f2b6c374a82cf076cfa1416992ac8e8ec94d79facc37aec87c1a5cb72352/cryptography-47.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2207a498b03275d0051589e326b79d4cf59985c99031b05bb292ac52631c37fe", size = 4688262, upload-time = "2026-04-24T19:54:26.946Z" }, - { url = "https://files.pythonhosted.org/packages/14/e2/8b7462f4acf21ec509616f0245018bb197194ab0b65c2ea21a0bdd53c0eb/cryptography-47.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a02675e2fabd0c0fc04c868b8781863cbf1967691543c22f5470500ff840b31", size = 4775506, upload-time = "2026-04-24T19:54:28.926Z" }, - { url = "https://files.pythonhosted.org/packages/70/75/158e494e4c08dc05e039da5bb48553826bd26c23930cf8d3cd5f21fa8921/cryptography-47.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80887c5cbd1774683cb126f0ab4184567f080071d5acf62205acb354b4b753b7", size = 4912060, upload-time = "2026-04-24T19:54:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/06/bd/0a9d3edbf5eadbac926d7b9b3cd0c4be584eeeae4a003d24d9eda4affbbd/cryptography-47.0.0-cp38-abi3-win32.whl", hash = "sha256:ed67ea4e0cfb5faa5bc7ecb6e2b8838f3807a03758eec239d6c21c8769355310", size = 3248487, upload-time = "2026-04-24T19:54:33.494Z" }, - { url = "https://files.pythonhosted.org/packages/60/80/5681af756d0da3a599b7bdb586fac5a1540f1bcefd2717a20e611ddade45/cryptography-47.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:835d2d7f47cdc53b3224e90810fb1d36ca94ea29cc1801fb4c1bc43876735769", size = 3755737, upload-time = "2026-04-24T19:54:35.408Z" }, -] - -[[package]] -name = "cycler" -version = "0.12.1" +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, + { url = "https://files.pythonhosted.org/packages/ec/9e/db72b3ae7fc9cfad53e630e56c6ae83b9b6ff0bf3718ffb8012d20b3aabf/cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7", size = 4013892, upload-time = "2026-06-12T20:02:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, + { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, + { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, + { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/93ecac273d3738939d023612ad12cca9a3740a5345d69fda04134c43fd96/cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61", size = 3799153, upload-time = "2026-06-12T20:01:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, +] + +[[package]] +name = "dask" +version = "2026.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +dependencies = [ + { name = "click" }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/ca/58434f10ebb45d2ddc6edd6e2988abcd38da1ab1897a5f6f402711a594e9/dask-2026.6.0.tar.gz", hash = "sha256:ae3436bd31ebce2be75edf952bd1fc687a1f11ec03fe8b1bec2903d222344a45", size = 11544529, upload-time = "2026-06-11T17:48:43.316Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, + { url = "https://files.pythonhosted.org/packages/02/17/b82b537a30be67ba178fc0b0fbda8fcbaeda188fb039d6ad8a84e89353a2/dask-2026.6.0-py3-none-any.whl", hash = "sha256:1539859071065dca379ca592ff76e911cd7965dc466da0040354ab466179189b", size = 1488995, upload-time = "2026-06-11T17:48:41.008Z" }, ] [[package]] name = "decorator" -version = "5.2.1" +version = "5.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, ] [[package]] name = "distlib" -version = "0.4.0" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, ] [[package]] -name = "earthengine-api" -version = "1.7.24" +name = "donfig" +version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-python-client" }, - { name = "google-auth" }, - { name = "google-auth-httplib2" }, - { name = "google-cloud-storage" }, - { name = "httplib2" }, - { name = "requests" }, + { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/10/cf255235e08ff8f2194f8d4559bc603261605f8c1d4948be33028e4658d7/earthengine_api-1.7.24.tar.gz", hash = "sha256:6ce804ad53276aed78a5f810a5a244b2da6225615905dcdcb6bc95f6f3c6fd2d", size = 437453, upload-time = "2026-04-27T18:32:34.252Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/f0/da043b6b151c697e1aebdb84049d31e3f35c64e2a322b74bfc5e86564b43/earthengine_api-1.7.24-py3-none-any.whl", hash = "sha256:9c7697d29b86b88eeeb9c88cc9cba21b6db1d648577bb0c57e714037b0afa627", size = 485811, upload-time = "2026-04-27T18:32:32.614Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] [[package]] -name = "eerepr" -version = "0.1.2" +name = "filelock" +version = "3.29.4" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "earthengine-api" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/f6/7c9cb681e477fbbec94359672e2a994e6cd3216912158675e0d1cdb1e455/eerepr-0.1.2.tar.gz", hash = "sha256:304dc23c365d6fa7cc3b07bc09cf77b565c19af86733e5deed804fdce16fb51f", size = 7695, upload-time = "2025-05-02T21:14:27.498Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/dc/be6cbe99670cd6e4ad387123647cb08e0c32975e223f82551e914c5568a6/filelock-3.29.4.tar.gz", hash = "sha256:10cdb3656fc44541cdf30652a93fb10ec6b05325620eb316bd26893e4201538a", size = 63028, upload-time = "2026-06-13T16:12:00.744Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/09/f92f3d87c967d80fb73fa45a7b8ce6048fcf6bc9ba04ef0fb04443e209d3/eerepr-0.1.2-py3-none-any.whl", hash = "sha256:f89731bde17de3d1e378b58a4cc9734f2c885014f56c2fe5eeab285d6eeb536c", size = 9517, upload-time = "2025-05-02T21:14:25.935Z" }, + { url = "https://files.pythonhosted.org/packages/13/37/a065dc3bd6e49423a6532c642ca7378d3f467b1ef44c2800c937af7f9739/filelock-3.29.4-py3-none-any.whl", hash = "sha256:dac1648087d5115554850d113e7dd8c83ab2d38e3435dde2d4f163847e57b767", size = 42757, upload-time = "2026-06-13T16:11:59.582Z" }, ] [[package]] -name = "executing" -version = "2.2.1" +name = "frozenlist" +version = "1.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, -] - -[[package]] -name = "filelock" -version = "3.29.0" +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/cf/b50ddf667c15276a9ab15a70ef5f257564de271957933ffea49d2cdbcdfb/fsspec-2026.3.0.tar.gz", hash = "sha256:1ee6a0e28677557f8c2f994e3eea77db6392b4de9cd1f5d7a9e87a0ae9d01b41", size = 313547, upload-time = "2026-03-27T19:11:14.892Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" }, ] [[package]] -name = "folium" -version = "0.20.0" +name = "gcsfs" +version = "2026.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "branca" }, - { name = "jinja2" }, - { name = "numpy" }, + { name = "aiohttp" }, + { name = "decorator" }, + { name = "fsspec" }, + { name = "google-auth" }, + { name = "google-auth-oauthlib" }, + { name = "google-cloud-storage" }, + { name = "google-cloud-storage-control" }, { name = "requests" }, - { name = "xyzservices" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/76/84a1b1b00ce71f9c0c44af7d80f310c02e2e583591fe7d4cb03baecd0d3f/folium-0.20.0.tar.gz", hash = "sha256:a0d78b9d5a36ba7589ca9aedbd433e84e9fcab79cd6ac213adbcff922e454cb9", size = 109932, upload-time = "2025-06-16T20:22:51.803Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/da/915029bc34b541c5ac3e6ba8b32ca14d278cd17588bbbb6e4b297a944cd9/gcsfs-2026.5.0.tar.gz", hash = "sha256:283941a7a53bdbfed2133f6b471c640e61c0a1379eb52280a26618b83acc49c0", size = 922614, upload-time = "2026-05-07T15:15:51.484Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/a8/5f764f333204db0390362a4356d03a43626997f26818a0e9396f1b3bd8c9/folium-0.20.0-py2.py3-none-any.whl", hash = "sha256:f0bc2a92acde20bca56367aa5c1c376c433f450608d058daebab2fc9bf8198bf", size = 113394, upload-time = "2025-06-16T20:22:50.318Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/7bed8e864afbb8f1870f8120dc4148ecec5480ba988d4241769a2b308a35/gcsfs-2026.5.0-py3-none-any.whl", hash = "sha256:a6abf1e6ee4b8ad7e0f137ca423c2f1441610ac98c3b933e11c505f8c62a90a9", size = 77744, upload-time = "2026-05-07T15:15:50.01Z" }, ] [[package]] -name = "fonttools" -version = "4.62.1" +name = "geopandas" +version = "1.1.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/08/7012b00a9a5874311b639c3920270c36ee0c445b69d9989a85e5c92ebcb0/fonttools-4.62.1.tar.gz", hash = "sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d", size = 3580737, upload-time = "2026-03-13T13:54:25.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/f0/2888cdac391807d68d90dcb16ef858ddc1b5309bfc6966195a459dd326e2/fonttools-4.62.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fa1d16210b6b10a826d71bed68dd9ec24a9e218d5a5e2797f37c573e7ec215ca", size = 2864442, upload-time = "2026-03-13T13:53:37.509Z" }, - { url = "https://files.pythonhosted.org/packages/4b/b2/e521803081f8dc35990816b82da6360fa668a21b44da4b53fc9e77efcd62/fonttools-4.62.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:aa69d10ed420d8121118e628ad47d86e4caa79ba37f968597b958f6cceab7eca", size = 2410901, upload-time = "2026-03-13T13:53:40.55Z" }, - { url = "https://files.pythonhosted.org/packages/00/a4/8c3511ff06e53110039358dbbdc1a65d72157a054638387aa2ada300a8b8/fonttools-4.62.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd13b7999d59c5eb1c2b442eb2d0c427cb517a0b7a1f5798fc5c9e003f5ff782", size = 4999608, upload-time = "2026-03-13T13:53:42.798Z" }, - { url = "https://files.pythonhosted.org/packages/28/63/cd0c3b26afe60995a5295f37c246a93d454023726c3261cfbb3559969bb9/fonttools-4.62.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d337fdd49a79b0d51c4da87bc38169d21c3abbf0c1aa9367eff5c6656fb6dae", size = 4912726, upload-time = "2026-03-13T13:53:45.405Z" }, - { url = "https://files.pythonhosted.org/packages/70/b9/ac677cb07c24c685cf34f64e140617d58789d67a3dd524164b63648c6114/fonttools-4.62.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d241cdc4a67b5431c6d7f115fdf63335222414995e3a1df1a41e1182acd4bcc7", size = 4951422, upload-time = "2026-03-13T13:53:48.326Z" }, - { url = "https://files.pythonhosted.org/packages/e6/10/11c08419a14b85b7ca9a9faca321accccc8842dd9e0b1c8a72908de05945/fonttools-4.62.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c05557a78f8fa514da0f869556eeda40887a8abc77c76ee3f74cf241778afd5a", size = 5060979, upload-time = "2026-03-13T13:53:51.366Z" }, - { url = "https://files.pythonhosted.org/packages/4e/3c/12eea4a4cf054e7ab058ed5ceada43b46809fce2bf319017c4d63ae55bb4/fonttools-4.62.1-cp314-cp314-win32.whl", hash = "sha256:49a445d2f544ce4a69338694cad575ba97b9a75fff02720da0882d1a73f12800", size = 2283733, upload-time = "2026-03-13T13:53:53.606Z" }, - { url = "https://files.pythonhosted.org/packages/6b/67/74b070029043186b5dd13462c958cb7c7f811be0d2e634309d9a1ffb1505/fonttools-4.62.1-cp314-cp314-win_amd64.whl", hash = "sha256:1eecc128c86c552fb963fe846ca4e011b1be053728f798185a1687502f6d398e", size = 2335663, upload-time = "2026-03-13T13:53:56.23Z" }, - { url = "https://files.pythonhosted.org/packages/42/c5/4d2ed3ca6e33617fc5624467da353337f06e7f637707478903c785bd8e20/fonttools-4.62.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1596aeaddf7f78e21e68293c011316a25267b3effdaccaf4d59bc9159d681b82", size = 2947288, upload-time = "2026-03-13T13:53:59.397Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e9/7ab11ddfda48ed0f89b13380e5595ba572619c27077be0b2c447a63ff351/fonttools-4.62.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8f8fca95d3bb3208f59626a4b0ea6e526ee51f5a8ad5d91821c165903e8d9260", size = 2449023, upload-time = "2026-03-13T13:54:01.642Z" }, - { url = "https://files.pythonhosted.org/packages/b2/10/a800fa090b5e8819942e54e19b55fc7c21fe14a08757c3aa3ca8db358939/fonttools-4.62.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee91628c08e76f77b533d65feb3fbe6d9dad699f95be51cf0d022db94089cdc4", size = 5137599, upload-time = "2026-03-13T13:54:04.495Z" }, - { url = "https://files.pythonhosted.org/packages/37/dc/8ccd45033fffd74deb6912fa1ca524643f584b94c87a16036855b498a1ed/fonttools-4.62.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f37df1cac61d906e7b836abe356bc2f34c99d4477467755c216b72aa3dc748b", size = 4920933, upload-time = "2026-03-13T13:54:07.557Z" }, - { url = "https://files.pythonhosted.org/packages/99/eb/e618adefb839598d25ac8136cd577925d6c513dc0d931d93b8af956210f0/fonttools-4.62.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92bb00a947e666169c99b43753c4305fc95a890a60ef3aeb2a6963e07902cc87", size = 5016232, upload-time = "2026-03-13T13:54:10.611Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5f/9b5c9bfaa8ec82def8d8168c4f13615990d6ce5996fe52bd49bfb5e05134/fonttools-4.62.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bdfe592802ef939a0e33106ea4a318eeb17822c7ee168c290273cbd5fabd746c", size = 5042987, upload-time = "2026-03-13T13:54:13.569Z" }, - { url = "https://files.pythonhosted.org/packages/90/aa/dfbbe24c6a6afc5c203d90cc0343e24bcbb09e76d67c4d6eef8c2558d7ba/fonttools-4.62.1-cp314-cp314t-win32.whl", hash = "sha256:b820fcb92d4655513d8402d5b219f94481c4443d825b4372c75a2072aa4b357a", size = 2348021, upload-time = "2026-03-13T13:54:16.98Z" }, - { url = "https://files.pythonhosted.org/packages/13/6f/ae9c4e4dd417948407b680855c2c7790efb52add6009aaecff1e3bc50e8e/fonttools-4.62.1-cp314-cp314t-win_amd64.whl", hash = "sha256:59b372b4f0e113d3746b88985f1c796e7bf830dd54b28374cd85c2b8acd7583e", size = 2414147, upload-time = "2026-03-13T13:54:19.416Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ba/56147c165442cc5ba7e82ecf301c9a68353cede498185869e6e02b4c264f/fonttools-4.62.1-py3-none-any.whl", hash = "sha256:7487782e2113861f4ddcc07c3436450659e3caa5e470b27dc2177cade2d8e7fd", size = 1152647, upload-time = "2026-03-13T13:54:22.735Z" }, +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyogrio" }, + { name = "pyproj" }, + { name = "shapely" }, ] - -[[package]] -name = "future" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ba/8e6b2091878e99e86a36a814dcaeff652ed48bdb03d53e78e15aaa63a914/geopandas-1.1.3.tar.gz", hash = "sha256:91a31989b6f566012838d21d5f8033f37dce882079ccb7cfdc40d5ccce7f284f", size = 336718, upload-time = "2026-03-09T21:49:09.545Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, + { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, ] [[package]] -name = "geemap" -version = "0.37.2" +name = "gitdb" +version = "4.0.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anywidget" }, - { name = "bqplot" }, - { name = "earthengine-api" }, - { name = "eerepr" }, - { name = "folium" }, - { name = "geocoder" }, - { name = "ipyevents" }, - { name = "ipyfilechooser" }, - { name = "ipyleaflet" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "plotly" }, - { name = "pyperclip" }, - { name = "pyshp" }, - { name = "python-box" }, - { name = "requests" }, - { name = "scooby" }, + { name = "smmap" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/b0/86dc7f7e6eeb15b70970e1e62bbedf0e90252502eed2cad95a1e0fc0e314/geemap-0.37.2.tar.gz", hash = "sha256:7e14102047079deed7b54a9e280f94693c6771fce9213cdf538abbff721d4c11", size = 2470580, upload-time = "2026-03-20T20:51:33.163Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/96/4f/fb3392a41254afc656c6dd1ea4b9936a012e451fbe5e934fbda81878a6c3/geemap-0.37.2-py3-none-any.whl", hash = "sha256:be717818f305317302d2be6fb7bbf7dde7dfe8bfa1af71325e745098678a764e", size = 2528877, upload-time = "2026-03-20T20:51:31.05Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, ] [[package]] -name = "geocoder" -version = "1.38.1" +name = "gitpython" +version = "3.1.50" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "future" }, - { name = "ratelim" }, - { name = "requests" }, - { name = "six" }, + { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/0b/2ea440270c1efb7ac73450cb704344c8127f45dabff0bea48711dc9dd93a/geocoder-1.38.1.tar.gz", hash = "sha256:c9925374c961577d0aee403b09e6f8ea1971d913f011f00ca70c76beaf7a77e7", size = 64345, upload-time = "2018-04-04T12:34:47.649Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl", hash = "sha256:a733e1dfbce3f4e1a526cac03aadcedb8ed1239cf55bd7f3a23c60075121a834", size = 98590, upload-time = "2018-04-04T12:34:51.222Z" }, + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, ] [[package]] name = "google-api-core" -version = "2.30.3" +version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-auth" }, @@ -511,9 +540,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/22/155cadf1d49272a9cf48f3168c0f3874fa13397297e611a5ea00cd093880/google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2", size = 176492, upload-time = "2026-06-03T14:52:17.257Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, + { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, ] [package.optional-dependencies] @@ -522,107 +551,77 @@ grpc = [ { name = "grpcio-status" }, ] -[[package]] -name = "google-api-python-client" -version = "2.194.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-auth-httplib2" }, - { name = "httplib2" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/60/ab/e83af0eb043e4ccc49571ca7a6a49984e9d00f4e9e6e6f1238d60bc84dce/google_api_python_client-2.194.0.tar.gz", hash = "sha256:db92647bd1a90f40b79c9618461553c2b20b6a43ce7395fa6de07132dc14f023", size = 14443469, upload-time = "2026-04-08T23:07:35.757Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/34/5a624e49f179aa5b0cb87b2ce8093960299030ff40423bfbde09360eb908/google_api_python_client-2.194.0-py3-none-any.whl", hash = "sha256:61eaaac3b8fc8fdf11c08af87abc3d1342d1b37319cc1b57405f86ef7697e717", size = 15016514, upload-time = "2026-04-08T23:07:33.093Z" }, -] - [[package]] name = "google-auth" -version = "2.49.2" +version = "2.54.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, -] - -[[package]] -name = "google-auth-httplib2" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "httplib2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/99/107612bef8d24b298bb5a7c8466f908ecda791d43f9466f5c3978f5b24c1/google_auth_httplib2-0.3.1.tar.gz", hash = "sha256:0af542e815784cb64159b4469aa5d71dd41069ba93effa006e1916b1dcd88e55", size = 11152, upload-time = "2026-03-30T22:50:26.766Z" } +sdist = { url = "https://files.pythonhosted.org/packages/15/f6/494e18317546d7def90c957b71d68b025d24f0e22e486c2606bc57765c48/google_auth-2.54.0.tar.gz", hash = "sha256:130f6fd5e3f497fdad897a23ed9489973437edf561238c4b92a4d02c435f8af9", size = 343161, upload-time = "2026-06-12T18:03:17.606Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/97/e9/93afb14d23a949acaa3f4e7cc51a0024671174e116e35f42850764b99634/google_auth_httplib2-0.3.1-py3-none-any.whl", hash = "sha256:682356a90ef4ba3d06548c37e9112eea6fc00395a11b0303a644c1a86abc275c", size = 9534, upload-time = "2026-03-30T22:49:03.384Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/d53bddd2c0949833fcb4ea06f9d5dd1c40575a1a4214cd1021eff57ba301/google_auth-2.54.0-py3-none-any.whl", hash = "sha256:784e9837f92244141250470d47c893df50cbab485ce491aca5e9deb558ad2b48", size = 249878, upload-time = "2026-06-12T18:02:57.58Z" }, ] [[package]] name = "google-auth-oauthlib" -version = "1.3.1" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-auth" }, { name = "requests-oauthlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/82/62482931dcbe5266a2680d0da17096f2aab983ecb320277d9556700ce00e/google_auth_oauthlib-1.3.1.tar.gz", hash = "sha256:14c22c7b3dd3d06dbe44264144409039465effdd1eef94f7ce3710e486cc4bfa", size = 21663, upload-time = "2026-03-30T22:49:56.408Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675, upload-time = "2026-05-07T08:03:47.816Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/e0/cb454a95f460903e39f101e950038ec24a072ca69d0a294a6df625cc1627/google_auth_oauthlib-1.3.1-py3-none-any.whl", hash = "sha256:1a139ef23f1318756805b0e95f655c238bffd29655329a2978218248da4ee7f8", size = 19247, upload-time = "2026-03-30T20:02:23.894Z" }, + { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261, upload-time = "2026-05-07T08:02:13.798Z" }, ] [[package]] -name = "google-cloud-bigquery" -version = "3.41.0" +name = "google-cloud-core" +version = "2.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core", extra = ["grpc"] }, + { name = "google-api-core" }, { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-resumable-media" }, - { name = "packaging" }, - { name = "python-dateutil" }, - { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ce/13/6515c7aab55a4a0cf708ffd309fb9af5bab54c13e32dc22c5acd6497193c/google_cloud_bigquery-3.41.0.tar.gz", hash = "sha256:2217e488b47ed576360c9b2cc07d59d883a54b83167c0ef37f915c26b01a06fe", size = 513434, upload-time = "2026-03-30T22:50:55.347Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/33/1d3902efadef9194566d499d61507e1f038454e0b55499d2d7f8ab2a4fee/google_cloud_bigquery-3.41.0-py3-none-any.whl", hash = "sha256:2a5b5a737b401cbd824a6e5eac7554100b878668d908e6548836b5d8aaa4dcaa", size = 262343, upload-time = "2026-03-30T22:48:45.444Z" }, + { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, ] [[package]] -name = "google-cloud-core" -version = "2.5.1" +name = "google-cloud-storage" +version = "3.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-core" }, { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/24/6ca08b0a03c7b0c620427503ab00353a4ae806b848b93bcea18b6b76fde6/google_cloud_core-2.5.1.tar.gz", hash = "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811", size = 36078, upload-time = "2026-03-30T22:50:08.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/72/86f94e1639a8bcd9d33e8e01b49afcaa1c3a13bda7683c681717e0901e15/google_cloud_storage-3.12.0.tar.gz", hash = "sha256:03ae9847c6babb368f35f054126b8a08cbc0e3266efb990eb17b9926a45cf3be", size = 17338620, upload-time = "2026-06-12T18:03:29.215Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/d9/5bb050cb32826466aa9b25f79e2ca2879fe66cb76782d4ed798dd7506151/google_cloud_core-2.5.1-py3-none-any.whl", hash = "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7", size = 29452, upload-time = "2026-03-30T22:48:31.567Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bd/a89eaebd2f9db5f92ddcc8e4f23c266be1dbd11058bb83451d8dd029f34c/google_cloud_storage-3.12.0-py3-none-any.whl", hash = "sha256:3880773754ddf7c27567b04e2a4d193950b6b99429f37b9097d873686e95b09c", size = 340605, upload-time = "2026-06-12T18:03:12.677Z" }, ] [[package]] -name = "google-cloud-storage" -version = "3.10.1" +name = "google-cloud-storage-control" +version = "1.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core" }, + { name = "google-api-core", extra = ["grpc"] }, { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, + { name = "grpc-google-iam-v1" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4c/47/205eb8e9a1739b5345843e5a425775cbdc472cc38e7eda082ba5b8d02450/google_cloud_storage-3.10.1.tar.gz", hash = "sha256:97db9aa4460727982040edd2bd13ff3d5e2260b5331ad22895802da1fc2a5286", size = 17309950, upload-time = "2026-03-23T09:35:23.409Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/ae/707995271f77e3c1da715c740160f98f87a79a5c601b28d17c4d2500c037/google_cloud_storage_control-1.12.0.tar.gz", hash = "sha256:49090d03532c0c84c6246a5fd490c31fd4bbffb4c7771c0a6744ec23a194a5f6", size = 137036, upload-time = "2026-06-03T16:14:00.921Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/ff/ca9ab2417fa913d75aae38bf40bf856bb2749a604b2e0f701b37cfcd23cc/google_cloud_storage-3.10.1-py3-none-any.whl", hash = "sha256:a72f656759b7b99bda700f901adcb3425a828d4a29f911bc26b3ea79c5b1217f", size = 324453, upload-time = "2026-03-23T09:35:21.368Z" }, + { url = "https://files.pythonhosted.org/packages/34/ba/f187f30fb2c8bb5dbb0de0fcb0dec2f8fab0b782ab42facc83ea02628f1b/google_cloud_storage_control-1.12.0-py3-none-any.whl", hash = "sha256:20f7f1252fa5635d47e12f746aa69d719e31eb9405cbbd14cdfba317db27d421", size = 102205, upload-time = "2026-06-03T16:12:43.266Z" }, ] [[package]] @@ -640,73 +639,80 @@ wheels = [ [[package]] name = "google-resumable-media" -version = "2.8.2" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-crc32c" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/d1/b1ea14b93b6b78f57fc580125de44e9f593ab88dd2460f1a8a8d18f74754/google_resumable_media-2.8.2.tar.gz", hash = "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70", size = 2164510, upload-time = "2026-03-30T23:34:25.369Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/f8/50bfaf4658431ff9de45c5c3935af7ab01157a4903c603cd0eee6e78e087/google_resumable_media-2.8.2-py3-none-any.whl", hash = "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220", size = 81511, upload-time = "2026-03-30T23:34:09.671Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, ] [[package]] name = "googleapis-common-protos" -version = "1.74.0" +version = "1.75.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, ] [[package]] -name = "grpcio" -version = "1.80.0" +name = "grpc-google-iam-v1" +version = "0.14.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "googleapis-common-protos", extra = ["grpc"] }, + { name = "grpcio" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/4f/d098419ad0bfc06c9ce440575f05aa22d8973b6c276e86ac7890093d3c37/grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038", size = 23706, upload-time = "2026-04-01T01:57:49.813Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/6d/e65307ce20f5a09244ba9e9d8476e99fb039de7154f37fb85f26978b59c3/grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e", size = 6017376, upload-time = "2026-03-30T08:48:10.005Z" }, - { url = "https://files.pythonhosted.org/packages/69/10/9cef5d9650c72625a699c549940f0abb3c4bfdb5ed45a5ce431f92f31806/grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f", size = 12018133, upload-time = "2026-03-30T08:48:12.927Z" }, - { url = "https://files.pythonhosted.org/packages/04/82/983aabaad82ba26113caceeb9091706a0696b25da004fe3defb5b346e15b/grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9", size = 6574748, upload-time = "2026-03-30T08:48:16.386Z" }, - { url = "https://files.pythonhosted.org/packages/07/d7/031666ef155aa0bf399ed7e19439656c38bbd143779ae0861b038ce82abd/grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14", size = 7277711, upload-time = "2026-03-30T08:48:19.627Z" }, - { url = "https://files.pythonhosted.org/packages/e8/43/f437a78f7f4f1d311804189e8f11fb311a01049b2e08557c1068d470cb2e/grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05", size = 6785372, upload-time = "2026-03-30T08:48:22.373Z" }, - { url = "https://files.pythonhosted.org/packages/93/3d/f6558e9c6296cb4227faa5c43c54a34c68d32654b829f53288313d16a86e/grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1", size = 7395268, upload-time = "2026-03-30T08:48:25.638Z" }, - { url = "https://files.pythonhosted.org/packages/06/21/0fdd77e84720b08843c371a2efa6f2e19dbebf56adc72df73d891f5506f0/grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f", size = 8392000, upload-time = "2026-03-30T08:48:28.974Z" }, - { url = "https://files.pythonhosted.org/packages/f5/68/67f4947ed55d2e69f2cc199ab9fd85e0a0034d813bbeef84df6d2ba4d4b7/grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e", size = 7828477, upload-time = "2026-03-30T08:48:32.054Z" }, - { url = "https://files.pythonhosted.org/packages/44/b6/8d4096691b2e385e8271911a0de4f35f0a6c7d05aff7098e296c3de86939/grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae", size = 4218563, upload-time = "2026-03-30T08:48:34.538Z" }, - { url = "https://files.pythonhosted.org/packages/e5/8c/bbe6baf2557262834f2070cf668515fa308b2d38a4bbf771f8f7872a7036/grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f", size = 5019457, upload-time = "2026-03-30T08:48:37.308Z" }, + { url = "https://files.pythonhosted.org/packages/89/22/c2dd50c09bf679bd38173656cd4402d2511e563b33bc88f90009cf50613c/grpc_google_iam_v1-0.14.4-py3-none-any.whl", hash = "sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964", size = 32675, upload-time = "2026-04-01T01:57:47.69Z" }, ] [[package]] -name = "grpcio-status" -version = "1.80.0" +name = "grpcio" +version = "1.81.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "protobuf" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" }, + { url = "https://files.pythonhosted.org/packages/b6/58/19414622b1bf6981bc9c05a365bd548e71876c89000083b3af489251e9c0/grpcio-1.81.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:506f48f2f9c29b143fca3dad7b0d518c188b6c9648c75a2ae6e2d9f2c13a060b", size = 6055336, upload-time = "2026-06-11T12:46:20.557Z" }, + { url = "https://files.pythonhosted.org/packages/32/f1/2ec88adb92b0eba970dd0e0e7dd086341daa3c75eba4f735f9e44bf684b0/grpcio-1.81.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d865db4a6318e1c1bea83292e0ed231090538fc4ca45425b0f0480eb338bbc6e", size = 12056279, upload-time = "2026-06-11T12:46:24.255Z" }, + { url = "https://files.pythonhosted.org/packages/41/36/e8c5f8c6ec71de73733695ebc809e98b178b534ec6d8eaa31a7ebab4ad4c/grpcio-1.81.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2aa72e3ce1770317ef534f63d397b55e130725f5149bd36077c3b539019db27", size = 6608225, upload-time = "2026-06-11T12:46:27.601Z" }, + { url = "https://files.pythonhosted.org/packages/30/22/96fc577a845ab093326d9ab1adb874bd4936c8cf98ac8ed2f3db13a0a2fb/grpcio-1.81.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0490c30c261eded63f3f354979f9dc4502a9fb944cccb60cd9dc85f5a7349854", size = 7306576, upload-time = "2026-06-11T12:46:30.514Z" }, + { url = "https://files.pythonhosted.org/packages/76/7b/61dab5d5969f28d97fb1009cead1df0a5cd987d3315e1b37f18a4449f8bc/grpcio-1.81.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:410482da976329fe5f4067270401b12cf2bd552ff8020f054ecfaddb5475f9d6", size = 6812165, upload-time = "2026-06-11T12:46:33.699Z" }, + { url = "https://files.pythonhosted.org/packages/82/78/6e501929d4f5f96462fd82fd9f0f06e5f9612207582b862868d68757b27d/grpcio-1.81.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3657301562ac3cb8018d30d0d3ebfa39932239f7b5703422057ef14b69949f5", size = 7422962, upload-time = "2026-06-11T12:46:36.511Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7e/f2157589e66daa78ebb3165942d05a08bdea93b9d11c2bc1e172aef89685/grpcio-1.81.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:24c8e57504c8f45b237e40b99262d181071e5099a07053695b75d97bb53053a0", size = 8408176, upload-time = "2026-06-11T12:46:39.803Z" }, + { url = "https://files.pythonhosted.org/packages/da/df/c6717fef716e00d235ffb96123baf6dce76d6004f6233fa767c502861460/grpcio-1.81.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b427c19380991a4eaab2f6144b64b99b412043314c6bf4ab544f97bb31ee4190", size = 7846681, upload-time = "2026-06-11T12:46:43.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/84/3502e9f210a6a5c4438c8aca3f88edd2e04f6a27f3d41b26cf0a0024b096/grpcio-1.81.1-cp314-cp314-win32.whl", hash = "sha256:61233fe8951e5c85dff81c2458b6528624760166946b5b47ea150a589168411f", size = 4264615, upload-time = "2026-06-11T12:46:45.741Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b0/4af731ff7492c68a96e4c71bfd0f4590acde92b31c6fe4894e6465c10ff6/grpcio-1.81.1-cp314-cp314-win_amd64.whl", hash = "sha256:3768a5ff1b2125e6f552e561b6b2dca0e64982d8949689b4df145cf8b98d7821", size = 5070275, upload-time = "2026-06-11T12:46:48.486Z" }, ] [[package]] -name = "httplib2" -version = "0.31.2" +name = "grpcio-status" +version = "1.81.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyparsing" }, + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } +sdist = { url = "https://files.pythonhosted.org/packages/32/26/0aa9168c87882381fd810d140c279a2490ed6aee655f0515d6f56c5ca404/grpcio_status-1.81.1.tar.gz", hash = "sha256:9389a03e746017b10f0630c064289201458f3ce01f5d7ef4b0bebc1ef6cf82ad", size = 13923, upload-time = "2026-06-11T12:58:48.636Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5e/5abfec5f7e89d3b7993d57cfb025ca5f968a2c18656d7fcda2b6919440b9/grpcio_status-1.81.1-py3-none-any.whl", hash = "sha256:08072fa9995f4a95c647fc6f4f85e2411573d00087bcabdf30f260114338f232", size = 14638, upload-time = "2026-06-11T12:58:31.982Z" }, ] [[package]] @@ -720,11 +726,11 @@ wheels = [ [[package]] name = "idna" -version = "3.13" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210, upload-time = "2026-04-22T16:42:42.314Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -736,112 +742,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "ipyevents" -version = "2.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipywidgets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/82/e5/ac7e199dd22ce3f7e0d43c6433fef5450197fa81994393e3381893b12d61/ipyevents-2.0.4.tar.gz", hash = "sha256:bd8c66a9ff26f481494cda1ec1d979722ca9eba19f8ef52538c7cc2db2a1a139", size = 210601, upload-time = "2025-09-15T14:02:34.29Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/d3/642a6dc3db8ea558a9b5fbc83815b197861868dc98f98a789b85c7660670/ipyevents-2.0.4-py3-none-any.whl", hash = "sha256:e532e05b037f70373850723f483d2830cb8a633e5aa19637ee9e7adaf41421f1", size = 102433, upload-time = "2025-09-15T14:02:33.255Z" }, -] - -[[package]] -name = "ipyfilechooser" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipywidgets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/63/a0bfdda0cdf002421632734e203e4692d72dc4e71cd006f101c90f9ba6d3/ipyfilechooser-0.6.0.tar.gz", hash = "sha256:41df9e4395a924f8e1b78e2804dbe5066dc3fdc233fb07fecfcdc2a0c9a7d8d3", size = 12737, upload-time = "2021-09-15T22:31:56.261Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/60/249e3444fcd9c833704741769981cd02fe2c7ce94126b1394e7a3b26e543/ipyfilechooser-0.6.0-py3-none-any.whl", hash = "sha256:4555c24b30b819c91dc0ae5e6f7e4cf8f90e5cca531a9209a1fe4deee288d5c5", size = 11039, upload-time = "2021-09-15T22:31:55.152Z" }, -] - -[[package]] -name = "ipyleaflet" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "branca" }, - { name = "ipywidgets" }, - { name = "jupyter-leaflet" }, - { name = "traittypes" }, - { name = "xyzservices" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d8/f2/00facc58a1e9581d9365df5761bf7e12f9fa7132132111f6ee484fd58e42/ipyleaflet-0.20.0.tar.gz", hash = "sha256:098f317dd63c4bcac5176d5b78d40d5758c623f7cd013f0d0c9d7a70cefcdb34", size = 28932, upload-time = "2025-06-13T08:33:43.441Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/69/e9858f2c0b99bf9f036348d1c84b8026f438bb6875effe6a9bcd9883dada/ipyleaflet-0.20.0-py3-none-any.whl", hash = "sha256:b4c20ddc0b17d68e226cd3367ca2215a4db7e2b14374468c0eeaa54b53e4d173", size = 31578, upload-time = "2025-06-13T08:33:37.353Z" }, -] - -[[package]] -name = "ipython" -version = "9.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/86/3060e8029b7cc505cce9a0137431dda81d0a3fde93a8f0f50ee0bf37a795/ipython-9.13.0-py3-none-any.whl", hash = "sha256:57f9d4639e20818d328d287c7b549af3d05f12486ea8f2e7f73e52a36ec4d201", size = 627274, upload-time = "2026-04-24T12:24:53.038Z" }, -] - -[[package]] -name = "ipython-pygments-lexers" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, -] - -[[package]] -name = "ipywidgets" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "comm" }, - { name = "ipython" }, - { name = "jupyterlab-widgets" }, - { name = "traitlets" }, - { name = "widgetsnbextension" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, -] - [[package]] name = "jdluc" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "earthengine-api" }, - { name = "geemap" }, - { name = "google-api-python-client" }, - { name = "google-auth-oauthlib" }, - { name = "google-cloud-bigquery" }, + { name = "dask" }, + { name = "gcsfs" }, + { name = "geopandas" }, + { name = "gitpython" }, { name = "google-cloud-storage" }, { name = "netcdf4" }, + { name = "networkx" }, { name = "pandas" }, + { name = "pyarrow" }, + { name = "python-dotenv" }, { name = "requests" }, + { name = "rio-cogeo" }, { name = "rioxarray" }, { name = "xarray" }, + { name = "zarr" }, ] [package.dev-dependencies] @@ -850,22 +770,28 @@ dev = [ { name = "pandas-stubs" }, { name = "pre-commit" }, { name = "pytest" }, + { name = "types-geopandas" }, + { name = "types-networkx" }, { name = "types-requests" }, ] [package.metadata] requires-dist = [ - { name = "earthengine-api" }, - { name = "geemap" }, - { name = "google-api-python-client" }, - { name = "google-auth-oauthlib" }, - { name = "google-cloud-bigquery" }, + { name = "dask" }, + { name = "gcsfs" }, + { name = "geopandas" }, + { name = "gitpython" }, { name = "google-cloud-storage" }, { name = "netcdf4" }, + { name = "networkx" }, { name = "pandas" }, + { name = "pyarrow" }, + { name = "python-dotenv" }, { name = "requests" }, + { name = "rio-cogeo" }, { name = "rioxarray" }, { name = "xarray" }, + { name = "zarr" }, ] [package.metadata.requires-dev] @@ -874,225 +800,142 @@ dev = [ { name = "pandas-stubs" }, { name = "pre-commit" }, { name = "pytest" }, + { name = "types-geopandas" }, + { name = "types-networkx" }, { name = "types-requests" }, ] -[[package]] -name = "jedi" -version = "0.19.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parso" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jupyter-leaflet" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/db/870bd0f8995c9359ea1501ba82ed7daf40bbbac95efeca80a35f3705929b/jupyter_leaflet-0.20.0.tar.gz", hash = "sha256:ad826dd7976a2b6d8b91d762c25a69a44f123b7b3bd1acaba236bd9af8e68cb4", size = 1306561, upload-time = "2025-06-13T08:33:46.282Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/95/ffe543060eb3b1570d78c3f2c1948c640a6758ff5c6479c27e474819115b/jupyter_leaflet-0.20.0-py3-none-any.whl", hash = "sha256:2e27ce83647316424f04845e3a6af35e1ee44c177c318a145646b11f4afe0764", size = 1107090, upload-time = "2025-06-13T08:33:40.416Z" }, -] - -[[package]] -name = "jupyterlab-widgets" -version = "3.0.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/2d/ef58fed122b268c69c0aa099da20bc67657cdfb2e222688d5731bd5b971d/jupyterlab_widgets-3.0.16.tar.gz", hash = "sha256:423da05071d55cf27a9e602216d35a3a65a3e41cdf9c5d3b643b814ce38c19e0", size = 897423, upload-time = "2025-11-01T21:11:29.724Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl", hash = "sha256:45fa36d9c6422cf2559198e4db481aa243c7a32d9926b500781c830c80f7ecf8", size = 914926, upload-time = "2025-11-01T21:11:28.008Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/d7/060f45052f2a01ad5762c8fdecd6d7a752b43400dc29ff75cd47225a40fd/kiwisolver-1.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615", size = 123231, upload-time = "2026-03-09T13:14:41.323Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a7/78da680eadd06ff35edef6ef68a1ad273bad3e2a0936c9a885103230aece/kiwisolver-1.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02", size = 66489, upload-time = "2026-03-09T13:14:42.534Z" }, - { url = "https://files.pythonhosted.org/packages/49/b2/97980f3ad4fae37dd7fe31626e2bf75fbf8bdf5d303950ec1fab39a12da8/kiwisolver-1.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e", size = 64063, upload-time = "2026-03-09T13:14:44.759Z" }, - { url = "https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac", size = 1475913, upload-time = "2026-03-09T13:14:46.247Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f0/f768ae564a710135630672981231320bc403cf9152b5596ec5289de0f106/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05", size = 1282782, upload-time = "2026-03-09T13:14:48.458Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9f/1de7aad00697325f05238a5f2eafbd487fb637cc27a558b5367a5f37fb7f/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd", size = 1300815, upload-time = "2026-03-09T13:14:50.721Z" }, - { url = "https://files.pythonhosted.org/packages/5a/c2/297f25141d2e468e0ce7f7a7b92e0cf8918143a0cbd3422c1ad627e85a06/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a", size = 1347925, upload-time = "2026-03-09T13:14:52.304Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d3/f4c73a02eb41520c47610207b21afa8cdd18fdbf64ffd94674ae21c4812d/kiwisolver-1.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554", size = 991322, upload-time = "2026-03-09T13:14:54.637Z" }, - { url = "https://files.pythonhosted.org/packages/7b/46/d3f2efef7732fcda98d22bf4ad5d3d71d545167a852ca710a494f4c15343/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581", size = 2232857, upload-time = "2026-03-09T13:14:56.471Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ec/2d9756bf2b6d26ae4349b8d3662fb3993f16d80c1f971c179ce862b9dbae/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303", size = 2329376, upload-time = "2026-03-09T13:14:58.072Z" }, - { url = "https://files.pythonhosted.org/packages/8f/9f/876a0a0f2260f1bde92e002b3019a5fabc35e0939c7d945e0fa66185eb20/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9", size = 1982549, upload-time = "2026-03-09T13:14:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4f/ba3624dfac23a64d54ac4179832860cb537c1b0af06024936e82ca4154a0/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79", size = 2494680, upload-time = "2026-03-09T13:15:01.364Z" }, - { url = "https://files.pythonhosted.org/packages/39/b7/97716b190ab98911b20d10bf92eca469121ec483b8ce0edd314f51bc85af/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796", size = 2297905, upload-time = "2026-03-09T13:15:03.925Z" }, - { url = "https://files.pythonhosted.org/packages/a3/36/4e551e8aa55c9188bca9abb5096805edbf7431072b76e2298e34fd3a3008/kiwisolver-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e", size = 75086, upload-time = "2026-03-09T13:15:07.775Z" }, - { url = "https://files.pythonhosted.org/packages/70/15/9b90f7df0e31a003c71649cf66ef61c3c1b862f48c81007fa2383c8bd8d7/kiwisolver-1.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df", size = 66577, upload-time = "2026-03-09T13:15:09.139Z" }, - { url = "https://files.pythonhosted.org/packages/17/01/7dc8c5443ff42b38e72731643ed7cf1ed9bf01691ae5cdca98501999ed83/kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e", size = 125794, upload-time = "2026-03-09T13:15:10.525Z" }, - { url = "https://files.pythonhosted.org/packages/46/8a/b4ebe46ebaac6a303417fab10c2e165c557ddaff558f9699d302b256bc53/kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4", size = 67646, upload-time = "2026-03-09T13:15:12.016Z" }, - { url = "https://files.pythonhosted.org/packages/60/35/10a844afc5f19d6f567359bf4789e26661755a2f36200d5d1ed8ad0126e5/kiwisolver-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028", size = 65511, upload-time = "2026-03-09T13:15:13.311Z" }, - { url = "https://files.pythonhosted.org/packages/f8/8a/685b297052dd041dcebce8e8787b58923b6e78acc6115a0dc9189011c44b/kiwisolver-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657", size = 1584858, upload-time = "2026-03-09T13:15:15.103Z" }, - { url = "https://files.pythonhosted.org/packages/9e/80/04865e3d4638ac5bddec28908916df4a3075b8c6cc101786a96803188b96/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920", size = 1392539, upload-time = "2026-03-09T13:15:16.661Z" }, - { url = "https://files.pythonhosted.org/packages/ba/01/77a19cacc0893fa13fafa46d1bba06fb4dc2360b3292baf4b56d8e067b24/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9", size = 1405310, upload-time = "2026-03-09T13:15:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/53/39/bcaf5d0cca50e604cfa9b4e3ae1d64b50ca1ae5b754122396084599ef903/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d", size = 1456244, upload-time = "2026-03-09T13:15:20.444Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7a/72c187abc6975f6978c3e39b7cf67aeb8b3c0a8f9790aa7fd412855e9e1f/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65", size = 1073154, upload-time = "2026-03-09T13:15:22.039Z" }, - { url = "https://files.pythonhosted.org/packages/c7/ca/cf5b25783ebbd59143b4371ed0c8428a278abe68d6d0104b01865b1bbd0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa", size = 2334377, upload-time = "2026-03-09T13:15:23.741Z" }, - { url = "https://files.pythonhosted.org/packages/4a/e5/b1f492adc516796e88751282276745340e2a72dcd0d36cf7173e0daf3210/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0", size = 2425288, upload-time = "2026-03-09T13:15:25.789Z" }, - { url = "https://files.pythonhosted.org/packages/e6/e5/9b21fbe91a61b8f409d74a26498706e97a48008bfcd1864373d32a6ba31c/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9", size = 2063158, upload-time = "2026-03-09T13:15:27.63Z" }, - { url = "https://files.pythonhosted.org/packages/b1/02/83f47986138310f95ea95531f851b2a62227c11cbc3e690ae1374fe49f0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f", size = 2597260, upload-time = "2026-03-09T13:15:29.421Z" }, - { url = "https://files.pythonhosted.org/packages/07/18/43a5f24608d8c313dd189cf838c8e68d75b115567c6279de7796197cfb6a/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646", size = 2394403, upload-time = "2026-03-09T13:15:31.517Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b5/98222136d839b8afabcaa943b09bd05888c2d36355b7e448550211d1fca4/kiwisolver-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681", size = 79687, upload-time = "2026-03-09T13:15:33.204Z" }, - { url = "https://files.pythonhosted.org/packages/99/a2/ca7dc962848040befed12732dff6acae7fb3c4f6fc4272b3f6c9a30b8713/kiwisolver-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57", size = 70032, upload-time = "2026-03-09T13:15:34.411Z" }, -] - [[package]] name = "librt" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/39/cb/c1945e506893b5b8577fb45a60c80e3ffe4a82092a04a6f29b0b951d9a24/librt-0.10.0.tar.gz", hash = "sha256:1aba1e8aa4e3307a7be68a74149545fde7451964dc0235a8bec5704a17bdda42", size = 191799, upload-time = "2026-05-05T16:31:23.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/64/7165e08108cc185a13a9c069f0685e6ef92e70e07fddf7edf5e7348c6316/librt-0.10.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:f09588a30e6a22ec624090d72a3ab1a6d4d5485c3ed739603e76aa3c16efa688", size = 76794, upload-time = "2026-05-05T16:30:20.392Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ef/bf8613febf651b90c5222ee79dea5ae58d4cc2b544df69d3033424448934/librt-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:131ade118d12bd7a0adc4e655474a553f1b76cf78385868885944d21d51e45e0", size = 79662, upload-time = "2026-05-05T16:30:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/b6/67/9eddd165c1d8397bdf99b38bf12b5a55b3def5035b49eedb49f2775d1430/librt-0.10.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8b9ab28e40d011c373a189eae900c916e66d6fbecf7983e9e4883089ee085ef", size = 242390, upload-time = "2026-05-05T16:30:23.51Z" }, - { url = "https://files.pythonhosted.org/packages/10/d1/d95da80334501866cd37004ab5d7483220d05862fab4b5405394f0264f0d/librt-0.10.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:67c39bb30da73bae1f293d1ed8bc2f8f6642649dd0928d3600aeff3041ac23d6", size = 232603, upload-time = "2026-05-05T16:30:25.198Z" }, - { url = "https://files.pythonhosted.org/packages/0c/fa/e6d64d28718bc1be4e1736fcb037ca1c4dfca927e7167df75a7d5215665e/librt-0.10.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c3273c6b774614f093c8927c2bf1b077d0fefde988fe98f46a333734e5597ab", size = 259187, upload-time = "2026-05-05T16:30:26.772Z" }, - { url = "https://files.pythonhosted.org/packages/72/3f/3fdb77e7f937dad59cfd76b720be7e7643400ec76b2da35befab8d66ba30/librt-0.10.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9dd7c1b86a4baa583ab5db977484b93a2c474e69e96ef3e9538387ea54229cb9", size = 251846, upload-time = "2026-05-05T16:30:28.56Z" }, - { url = "https://files.pythonhosted.org/packages/18/ca/f4d49133dd86a6f55d79eca30bf412fa722f511a9abe67f62f57aa64e66a/librt-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a77385c5a202e831149f7ad03be9e67cf80e957e52c614e83dcb822c95222eb8", size = 264936, upload-time = "2026-05-05T16:30:30.491Z" }, - { url = "https://files.pythonhosted.org/packages/de/66/a8df2fbadc1f6c1827a096d11c40175bd526133480bd3bc88ec64a03d257/librt-0.10.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c6a5eafa74b5655bad59886138ed68426f098a6beb8cb95a71f2cc3cd8bb33fe", size = 258699, upload-time = "2026-05-05T16:30:32.002Z" }, - { url = "https://files.pythonhosted.org/packages/bb/73/1e3c83613fe05451bb969e27b68a573d177f08d5f63533cc29fec0989658/librt-0.10.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1fc93d0439204c50ab4d1512611ce2c206f1b369b419f69c7c27c761561e3291", size = 259825, upload-time = "2026-05-05T16:30:35.077Z" }, - { url = "https://files.pythonhosted.org/packages/09/24/5e2f926ee9d3ef348d9339526d7062abb5c44d8419e3179528c01d78c102/librt-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:79e713c178bc7a744adfbee6b4619a288eecc0c914da2a9313a20255abe2f0cf", size = 282548, upload-time = "2026-05-05T16:30:36.639Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7d/3e89ed6ad0162561fa8bef9df3195e24263104c955713cd0237d3711fad2/librt-0.10.0-cp314-cp314-win32.whl", hash = "sha256:2eba9d955a68c41d9f326be3da42f163ec3518b7ab20f1c826224e7bed71e0bf", size = 58970, upload-time = "2026-05-05T16:30:38.183Z" }, - { url = "https://files.pythonhosted.org/packages/76/25/579e731c94a7086a268bfa3e7a4945cd47836bebd3cbf3faeafd2e7eaef9/librt-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbfaf7f5145e9917f5d18bffa298eff6a19d74e7b8b11dabdca95785befe8dbf", size = 67260, upload-time = "2026-05-05T16:30:39.804Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f8/235822b7ae0b2334f12ee18bcf2476d07924077a5efeea57dbe927704be2/librt-0.10.0-cp314-cp314-win_arm64.whl", hash = "sha256:8d6d385d1969849a6b1397114df22714b6ded917bada98668e3e974dc663477e", size = 57156, upload-time = "2026-05-05T16:30:41.412Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e3/9b919cbf1e8eb770bf91bb7df28125e0f1daf4587169afefd95402636e9a/librt-0.10.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:6c3a82d3bd32631ef5c79922dfc028520c9ad840255979ab4d908271818039ee", size = 79150, upload-time = "2026-05-05T16:30:42.761Z" }, - { url = "https://files.pythonhosted.org/packages/6a/f5/72a944aa3bc3498169a168087eff58ca48b58bf1b704e59d091fd30739f3/librt-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d64cc66005dc324c9bb1fa3fc2841f529002f6eb15966d55e46d430f56955a6a", size = 82304, upload-time = "2026-05-05T16:30:44.082Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e3/fcc290a33e295019759472dfa794d204e43504b276ac65eab7fd9da20ea3/librt-0.10.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9bb562cd28c88cd2c6a9a6c78f99dc39348d6b16c94adc25de0e574acf1176e9", size = 272556, upload-time = "2026-05-05T16:30:45.497Z" }, - { url = "https://files.pythonhosted.org/packages/fd/54/546975e4c997573885e7f040a05012f8838e06fb12b0c3c1fbb76254e9d7/librt-0.10.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:b809aa2854d019c28773b03605df22adc675ee4f3f4402d673581313e8906119", size = 256941, upload-time = "2026-05-05T16:30:47.059Z" }, - { url = "https://files.pythonhosted.org/packages/70/8c/f1d03401571b331653acddbd4e8cd955c06d945241dd08b25192fac0d04b/librt-0.10.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc15acabdd519bd4176fdadc2119e5e3093485d86f89138daf47e5b4cedb983a", size = 285855, upload-time = "2026-05-05T16:30:48.86Z" }, - { url = "https://files.pythonhosted.org/packages/0c/08/62cf80ff046c339faf56718b3a940244d4beb70f1c6407289b5830ec11e9/librt-0.10.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b1b2d835307d08ddadd94568e2369648ec9173bd3eea6d7f52a1abe717c81f98", size = 275321, upload-time = "2026-05-05T16:30:50.63Z" }, - { url = "https://files.pythonhosted.org/packages/d9/ea/da5918d4070362e9a4d2ee9cd34f9dc84902daad8fd4275f8504a727ff4e/librt-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d261c6a2f93335a5167887fb0223e8b98ffce20ee3fde242e8e58a37ece6d0e5", size = 293993, upload-time = "2026-05-05T16:30:52.577Z" }, - { url = "https://files.pythonhosted.org/packages/c9/8d/68b6086bed1fcdc314c640ea04e31e52d18052e08059fa595409d66a51a9/librt-0.10.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e2ffd44963f8e7f68995504d90f9881d64e94dc1d8e310039b9526108fc0c0f7", size = 284254, upload-time = "2026-05-05T16:30:55.086Z" }, - { url = "https://files.pythonhosted.org/packages/06/c8/b810f1d84ec34a5a7ed93d7b510ab04164d75fbdf23088d5c3fbe6b08357/librt-0.10.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f285f6455ed495791c4d8630e5af732960adea93cac4c893d15619f2eae53e8", size = 284925, upload-time = "2026-05-05T16:30:56.728Z" }, - { url = "https://files.pythonhosted.org/packages/5a/00/3c82d4158c5a2c62528b8fccce65a8c9ad700e480e86f9389387435089a5/librt-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f6034ff52e663d34c7b82ef2aa2f94ad7c1d939e2368e63b06844bc4d127d2e1", size = 307830, upload-time = "2026-05-05T16:30:58.377Z" }, - { url = "https://files.pythonhosted.org/packages/99/3a/9c635ac3e8a00383ff689161d3eac8a30b3b2ddc711b40471e6b8983ea29/librt-0.10.0-cp314-cp314t-win32.whl", hash = "sha256:657860fd877fba6a241ea088ef99f63ca819945d3c715265da670bad56c37ebe", size = 60147, upload-time = "2026-05-05T16:31:00.293Z" }, - { url = "https://files.pythonhosted.org/packages/dc/e8/6f65f3e565d4ac212cddddd552eacc8035ffdf941ca0ad6fe945a211d41f/librt-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:56ded2d66010203a0cb5af063b609e3f079531a0e5e576d618dece859fd2e1af", size = 68649, upload-time = "2026-05-05T16:31:01.778Z" }, - { url = "https://files.pythonhosted.org/packages/51/78/a0705a67cacd81e5fa01a5035b3adbdfbb43a7b8d4bd27e2b282ae61baf2/librt-0.10.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1ee63f30abf18ed4830fdbaf87b2b6f4bba1e198d46085c314edde4045e56715", size = 58247, upload-time = "2026-05-05T16:31:03.191Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, - { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, - { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, - { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, - { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, - { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, - { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, - { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, - { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, - { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, - { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, - { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, - { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, - { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, - { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, - { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, - { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, - { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, -] - -[[package]] -name = "matplotlib" -version = "3.10.9" +sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/eb/dbce197da4e227779e56b5735f2decc3eb36e55a1cdbf1bd65d6639d76c1/librt-0.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd", size = 143345, upload-time = "2026-05-10T18:16:30.674Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/254bebd0c11c8ba684018efb8006ff22e466abce445215cca6c778e7d9de/librt-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4", size = 143131, upload-time = "2026-05-10T18:16:32.037Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3f/f77d6122d21ac7bf6ae8a7dfced1bd2a7ac545d3273ebdcaf8042f6d619f/librt-0.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8", size = 477024, upload-time = "2026-05-10T18:16:33.493Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0a/2c996dadebaa7d9bbbd43ef2d4f3e66b6da545f838a41694ef6172cebec8/librt-0.11.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b", size = 474221, upload-time = "2026-05-10T18:16:34.864Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7e/f5d92af8486b8272c23b3e686b46ff72d89c8169585eb61eef01a2ac7147/librt-0.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175", size = 505174, upload-time = "2026-05-10T18:16:36.705Z" }, + { url = "https://files.pythonhosted.org/packages/af/1a/cb0734fe86398eb33193ab753b7326255c74cac5eb09e76b9b16536e7adb/librt-0.11.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03", size = 497216, upload-time = "2026-05-10T18:16:38.418Z" }, + { url = "https://files.pythonhosted.org/packages/18/06/094820f91558b66e29943c0ec41c9914f460f48dd51fc503c3101e10842d/librt-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c", size = 513921, upload-time = "2026-05-10T18:16:39.848Z" }, + { url = "https://files.pythonhosted.org/packages/0b/c2/00de9018871a282f530cacb457d5ec0428f6ac7e6fedde9aff7468d9fb04/librt-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3", size = 520850, upload-time = "2026-05-10T18:16:41.471Z" }, + { url = "https://files.pythonhosted.org/packages/51/9d/64631832348fd1834fb3a61b996434edddaaf25a31d03b0a76273159d2cf/librt-0.11.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96", size = 504237, upload-time = "2026-05-10T18:16:43.15Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/ae5525eb16edc827a044e7bb8777a455ff95d4bca9379e7e6bddd7383647/librt-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe", size = 546261, upload-time = "2026-05-10T18:16:44.408Z" }, + { url = "https://files.pythonhosted.org/packages/5a/09/adce371f27ca039411da9659f7430fcc2ba6cd0c7b3e4467a0f091be7fa9/librt-0.11.0-cp314-cp314-win32.whl", hash = "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f", size = 96965, upload-time = "2026-05-10T18:16:46.039Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ee/8ac720d98548f173c7ce2e632a7ca94673f74cacd5c8162a84af5b35958a/librt-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7", size = 115151, upload-time = "2026-05-10T18:16:47.133Z" }, + { url = "https://files.pythonhosted.org/packages/94/20/c900cf14efeb09b6bef2b2dff20779f73464b97fd58d1c6bccc379588ae3/librt-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1", size = 98850, upload-time = "2026-05-10T18:16:48.597Z" }, + { url = "https://files.pythonhosted.org/packages/0c/71/944bfe4b64e12abffcd3c15e1cce07f72f3d55655083786285f4dedeb532/librt-0.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72", size = 151138, upload-time = "2026-05-10T18:16:49.839Z" }, + { url = "https://files.pythonhosted.org/packages/b6/10/99e64a5c86989357fda078c8143c533389585f6473b7439172dd8f3b3b2d/librt-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa", size = 151976, upload-time = "2026-05-10T18:16:51.062Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/5072ad880946d83e5ea4147d6d018c78eefce85b77819b19bdd0ee229435/librt-0.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548", size = 557927, upload-time = "2026-05-10T18:16:52.632Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8d/70b5fb7cfbab60edbe7381614ab985da58e144fbf465c86d44c95f43cdca/librt-0.11.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2", size = 539698, upload-time = "2026-05-10T18:16:53.934Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a3/ba3495a0b3edbd24a4cae0d1d3c64f39a9fc45d06e812101289b50c1a619/librt-0.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f", size = 577162, upload-time = "2026-05-10T18:16:55.589Z" }, + { url = "https://files.pythonhosted.org/packages/f7/db/36e25fb81f99937ff1b96612a1dc9fd66f039cb9cc3aee12c01fac31aab9/librt-0.11.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51", size = 566494, upload-time = "2026-05-10T18:16:56.975Z" }, + { url = "https://files.pythonhosted.org/packages/33/0d/3f622b47f0b013eeb9cf4cc07ae9bfe378d832a4eec998b2b209fe84244d/librt-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2", size = 596858, upload-time = "2026-05-10T18:16:58.374Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/71b90bc93039c46a2000651f6ad60122b114c8f54c4ad306e0e96f5b75ad/librt-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085", size = 590318, upload-time = "2026-05-10T18:16:59.676Z" }, + { url = "https://files.pythonhosted.org/packages/04/04/418cb3f75621e2b761fb1ab0f017f4d70a1a72a6e7c74ee4f7e8d198c2f3/librt-0.11.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3", size = 575115, upload-time = "2026-05-10T18:17:01.007Z" }, + { url = "https://files.pythonhosted.org/packages/cc/2c/5a2183ac58dd911f26b5d7e7d7d8f1d87fcecdddd99d6c12169a258ff62c/librt-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd", size = 617918, upload-time = "2026-05-10T18:17:02.682Z" }, + { url = "https://files.pythonhosted.org/packages/15/1f/dc6771a52592a4451be6effa200cbfc9cec61e4393d3033d81a9d307961d/librt-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8", size = 103562, upload-time = "2026-05-10T18:17:03.99Z" }, + { url = "https://files.pythonhosted.org/packages/62/4a/7d1415567027286a75ba1093ec4aca11f073e0f559c530cf3e0a757ad55c/librt-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c", size = 124327, upload-time = "2026-05-10T18:17:05.465Z" }, + { url = "https://files.pythonhosted.org/packages/ce/62/b40b382fa0c66fee1478073eb8db352a4a6beda4a1adccf1df911d8c289c/librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", size = 102572, upload-time = "2026-05-10T18:17:06.809Z" }, +] + +[[package]] +name = "locket" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/1b/4be5be87d43d327a0cf4de1a56e86f7f84c89312452406cf122efe2839e6/matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358", size = 34811233, upload-time = "2026-04-24T00:14:13.539Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/e6/3bd8afd04949f02eabc1c17115ea5255e19cacd4d06fc5abdde4eeb0052c/matplotlib-3.10.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:172db52c9e683f5d12eaf57f0f54834190e12581fe1cc2a19595a8f5acb4e77d", size = 8321276, upload-time = "2026-04-24T00:13:18.318Z" }, - { url = "https://files.pythonhosted.org/packages/41/86/86231232fff41c9f8e4a1a7d7a597d349a02527109c3af7d618366122139/matplotlib-3.10.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97e35e8d39ccc85859095e01a53847432ba9a53ddf7986f7a54a11b73d0e143f", size = 8218218, upload-time = "2026-04-24T00:13:20.974Z" }, - { url = "https://files.pythonhosted.org/packages/85/8f/becc9722cafc64f5d2eb0b7c1bf5f585271c618a45dbd8fabeb021f898b6/matplotlib-3.10.9-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aba1615dabe83188e19d4f75a253c6a08423e04c1425e64039f800050a69de6b", size = 9608145, upload-time = "2026-04-24T00:13:23.228Z" }, - { url = "https://files.pythonhosted.org/packages/32/5d/f7e914f7d9325abff4057cee62c0fa70263683189f774473cbfb534cd13b/matplotlib-3.10.9-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34cf8167e023ad956c15f36302911d5406bd99a9862c1a8499ea6f7c0e015dc2", size = 9885085, upload-time = "2026-04-24T00:13:25.849Z" }, - { url = "https://files.pythonhosted.org/packages/a5/fd/fa69f2221534e80cc5772ac2b7d222011a2acafc2ec7216d5dd174c864ae/matplotlib-3.10.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59476c6d29d612b8e9bb6ce8c5b631be6ba8f9e3a2421f22a02b192c7dd28716", size = 9672358, upload-time = "2026-04-24T00:13:28.906Z" }, - { url = "https://files.pythonhosted.org/packages/ab/1a/5a4f747a8b271cbb024946d2dd3c913ab5032ba430626f8c3528ada96b4b/matplotlib-3.10.9-cp314-cp314-win_amd64.whl", hash = "sha256:336b9acc64d309063126edcdaca00db9373af3c476bb94388fe9c5a53ad13e6f", size = 8349970, upload-time = "2026-04-24T00:13:31.904Z" }, - { url = "https://files.pythonhosted.org/packages/64/dc/95d60ecaefe30680a154b52ea96ab4b0dab547f1fd6aa12f5fb655e89cae/matplotlib-3.10.9-cp314-cp314-win_arm64.whl", hash = "sha256:2dc9477819ffd78ad12a20df1d9d6a6bd4fec6aaa9072681465fddca052f1456", size = 8272785, upload-time = "2026-04-24T00:13:34.511Z" }, - { url = "https://files.pythonhosted.org/packages/70/a0/005d68bc8b8418300ce6591f18586910a8526806e2ab663933d9f20a41e9/matplotlib-3.10.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:da4e09638420548f31c354032a6250e473c68e5a4e96899b4844cf39ddea23fe", size = 8367999, upload-time = "2026-04-24T00:13:36.962Z" }, - { url = "https://files.pythonhosted.org/packages/22/05/1236cc9290be70b2498af20ca348add76e3fffe7f67b477db5133a84f3ea/matplotlib-3.10.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:345f6f68ecc8da0ca56fad2ea08fde1a115eda530079eca185d50a7bc3e146c6", size = 8264543, upload-time = "2026-04-24T00:13:39.851Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c2/071f5a5ff6c5bd63aaaf2f45c811d9bf2ced94bde188d9e1a519e21d0cba/matplotlib-3.10.9-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4edcfbd8565339aa62f1cd4012f7180926fdbe71850f7b0d3c379c175cd6b66c", size = 9622800, upload-time = "2026-04-24T00:13:42.296Z" }, - { url = "https://files.pythonhosted.org/packages/95/57/da7d1f10a85624b9e7db68e069dd94e58dc41dbf9463c5921632ecbe3661/matplotlib-3.10.9-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6be157fe17fc37cb95ac1d7374cf717ce9259616edec911a78d9d26dae8522d4", size = 9888561, upload-time = "2026-04-24T00:13:45.026Z" }, - { url = "https://files.pythonhosted.org/packages/67/b2/ef8d6bb59b0edb6c16c968b70f548aa13b54348972def5aa6ac85df67145/matplotlib-3.10.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4e42042d54db34fda4e95a7bd3e5789c2a995d2dad3eb8850232ee534092fbbf", size = 9680884, upload-time = "2026-04-24T00:13:48.066Z" }, - { url = "https://files.pythonhosted.org/packages/61/1c/d21bfeb9931881ebe96bcfcff27c7ae4b160ae0ec291a714c42641a56d75/matplotlib-3.10.9-cp314-cp314t-win_amd64.whl", hash = "sha256:c27df8b3848f32a83d1767566595e43cfaa4460380974da06f4279a7ec143c39", size = 8432333, upload-time = "2026-04-24T00:13:51.008Z" }, - { url = "https://files.pythonhosted.org/packages/78/23/92493c3e6e1b635ccfff146f7b99e674808787915420373ac399283764c2/matplotlib-3.10.9-cp314-cp314t-win_arm64.whl", hash = "sha256:a49f1eadc84ca85fd72fa4e89e70e61bf86452df6f971af04b12c60761a0772c", size = 8324785, upload-time = "2026-04-24T00:13:53.633Z" }, + { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, ] [[package]] -name = "matplotlib-inline" -version = "0.2.1" +name = "morecantile" +version = "7.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets" }, + { name = "attrs" }, + { name = "click" }, + { name = "pydantic" }, + { name = "pyproj" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/74/97e72a36efd4ae2bccb3463284300f8953f199b5ffbc04cbbb0ec78f74b1/matplotlib_inline-0.2.1.tar.gz", hash = "sha256:e1ee949c340d771fc39e241ea75683deb94762c8fa5f2927ec57c83c4dffa9fe", size = 8110, upload-time = "2025-10-23T09:00:22.126Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ab/44/fa4f0685481b51e5ce33089ca84e821453593d34f306738515a937934751/morecantile-7.0.3.tar.gz", hash = "sha256:b44419c83f310b411b1f547df2d07c115dc6d194ab7c8a4c318a154490e938c1", size = 43104, upload-time = "2026-02-04T23:03:26.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/1a/f9edb1c167bd8ad214dc87e9ae17fb208d08f7a05e724f74ce29e375c8de/morecantile-7.0.3-py3-none-any.whl", hash = "sha256:747c6b8f3a8029ddaadb04d96c834f10d2796d1898ad893bed47c896a058ddc7", size = 50885, upload-time = "2026-02-04T23:03:27.463Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] [[package]] name = "mypy" -version = "1.20.2" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "ast-serialize" }, { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/af/e3d4b3e9ec91a0ff9aabfdb38692952acf49bbb899c2e4c29acb3a6da3ae/mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665", size = 3817349, upload-time = "2026-04-21T17:12:28.473Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/15/cca9d88503549ed6fedeaa1d448cdddd542ee8a490232d732e278036fbf2/mypy-2.1.0.tar.gz", hash = "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", size = 3898359, upload-time = "2026-05-11T18:37:36.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/d1/b4ec96b0ecc620a4443570c6e95c867903428cfcde4206518eafdd5880c3/mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30", size = 14524561, upload-time = "2026-04-21T17:06:27.325Z" }, - { url = "https://files.pythonhosted.org/packages/3a/63/d2c2ff4fa66bc49477d32dfa26e8a167ba803ea6a69c5efb416036909d30/mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924", size = 13363883, upload-time = "2026-04-21T17:11:11.239Z" }, - { url = "https://files.pythonhosted.org/packages/2a/56/983916806bf4eddeaaa2c9230903c3669c6718552a921154e1c5182c701f/mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb", size = 13742945, upload-time = "2026-04-21T17:08:34.181Z" }, - { url = "https://files.pythonhosted.org/packages/19/65/0cd9285ab010ee8214c83d67c6b49417c40d86ce46f1aa109457b5a9b8d7/mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc", size = 14706163, upload-time = "2026-04-21T17:05:15.51Z" }, - { url = "https://files.pythonhosted.org/packages/94/97/48ff3b297cafcc94d185243a9190836fb1b01c1b0918fff64e941e973cc9/mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558", size = 14938677, upload-time = "2026-04-21T17:05:39.562Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a1/1b4233d255bdd0b38a1f284feeb1c143ca508c19184964e22f8d837ec851/mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8", size = 11089322, upload-time = "2026-04-21T17:06:44.29Z" }, - { url = "https://files.pythonhosted.org/packages/78/c2/ce7ee2ba36aeb954ba50f18fa25d9c1188578654b97d02a66a15b6f09531/mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3", size = 10017775, upload-time = "2026-04-21T17:07:20.732Z" }, - { url = "https://files.pythonhosted.org/packages/4e/a1/9d93a7d0b5859af0ead82b4888b46df6c8797e1bc5e1e262a08518c6d48e/mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609", size = 15549002, upload-time = "2026-04-21T17:08:23.107Z" }, - { url = "https://files.pythonhosted.org/packages/00/d2/09a6a10ee1bf0008f6c144d9676f2ca6a12512151b4e0ad0ff6c4fac5337/mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2", size = 14401942, upload-time = "2026-04-21T17:07:31.837Z" }, - { url = "https://files.pythonhosted.org/packages/57/da/9594b75c3c019e805250bed3583bdf4443ff9e6ef08f97e39ae308cb06f2/mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c", size = 15041649, upload-time = "2026-04-21T17:09:34.653Z" }, - { url = "https://files.pythonhosted.org/packages/97/77/f75a65c278e6e8eba2071f7f5a90481891053ecc39878cc444634d892abe/mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744", size = 15864588, upload-time = "2026-04-21T17:11:44.936Z" }, - { url = "https://files.pythonhosted.org/packages/d7/46/1a4e1c66e96c1a3246ddf5403d122ac9b0a8d2b7e65730b9d6533ba7a6d3/mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6", size = 16093956, upload-time = "2026-04-21T17:10:17.683Z" }, - { url = "https://files.pythonhosted.org/packages/5a/2c/78a8851264dec38cd736ca5b8bc9380674df0dd0be7792f538916157716c/mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec", size = 12568661, upload-time = "2026-04-21T17:11:54.473Z" }, - { url = "https://files.pythonhosted.org/packages/83/01/cd7318aa03493322ce275a0e14f4f52b8896335e4e79d4fb8153a7ad2b77/mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382", size = 10389240, upload-time = "2026-04-21T17:09:42.719Z" }, - { url = "https://files.pythonhosted.org/packages/28/9a/f23c163e25b11074188251b0b5a0342625fc1cdb6af604757174fa9acc9b/mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563", size = 2637314, upload-time = "2026-04-21T17:05:54.5Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ca/b279a672e874aedd5498ae25f722dacc8aa86bbffb939b3f97cbb1cf6686/mypy-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7354c5a7f69d9345c3d6e69921d57088eea3ddeeb6b20d34c1b3855b02c36ec2", size = 14848422, upload-time = "2026-05-11T18:35:45.984Z" }, + { url = "https://files.pythonhosted.org/packages/27/e6/3efe56c631d959b9b4454e208b0ac4b7f4f58b404c89f8bec7b49efdfc21/mypy-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:49890d4f76ac9e06ec117f9e09f3174da70a620a0c300953d8595c926e80947f", size = 13677374, upload-time = "2026-05-11T18:36:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/84/7f/8107ea87a44fd1f1b59882442f033c9c3488c127201b1d1d15f1cbd6022e/mypy-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:761be68e023ef5d94678772396a8af1220030f80837a3afd8d0aef3b419666f4", size = 14055743, upload-time = "2026-05-11T18:35:18.361Z" }, + { url = "https://files.pythonhosted.org/packages/51/4d/b6d34db183133b83761b9199a82d31557cdbb70a380d8c3b3438e11882a3/mypy-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c90345fc182dc363b891350457ec69c35140858538f38b4540845afcc32b1aef", size = 15020937, upload-time = "2026-05-11T18:34:59.618Z" }, + { url = "https://files.pythonhosted.org/packages/ff/d7/f08360c691d758acb02f45022c34d98b92892f4ea756644e1000d4b9f3d8/mypy-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b84802e7b5a6daf1f5e15bc9fcd7ddae77be13981ffab037f1c67bb84d67d135", size = 15253371, upload-time = "2026-05-11T18:36:41.081Z" }, + { url = "https://files.pythonhosted.org/packages/67/1b/09460a13719530a19bce27bd3bc8449e83569dd2ba7faf51c9c3c30c0b61/mypy-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:022c771234936ceac541ebaf836fe9e2abeb3f5e09aff21588fe543ff006fe21", size = 11326429, upload-time = "2026-05-11T18:34:13.526Z" }, + { url = "https://files.pythonhosted.org/packages/40/62/75dbf0f82f7b6680340efc614af29dd0b3c17b8a4f1cd09b8bd2fd6bc814/mypy-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:498207db725cec88829a6a5c2fc771205fd043719ef98bc49aba8fb9fc4e6d57", size = 10218799, upload-time = "2026-05-11T18:32:23.491Z" }, + { url = "https://files.pythonhosted.org/packages/b2/66/caca04ed7d972fb6eb6dd1ccd6df1de5c38fae8c5b3dc1c4e8e0d85ee6b9/mypy-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7d5e5cad0efeba72b93cd17490cc0d69c5ac9ca132994fe3fb0314808aeeb83e", size = 15923458, upload-time = "2026-05-11T18:35:28.64Z" }, + { url = "https://files.pythonhosted.org/packages/ed/52/2d90cbe49d014b13ed7ff337930c30bad35893fe38a1e4641e756bb62191/mypy-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff715050c127d724fd260a2e666e7747fdd83511c0c47d449d98238970aef780", size = 14757697, upload-time = "2026-05-11T18:36:14.208Z" }, + { url = "https://files.pythonhosted.org/packages/ac/37/d98f4a14e081b238992d0ed96b6d39c7cc0148c9699eb71eaa68629665ea/mypy-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82208da9e09414d520e912d3e462d454854bed0810b71540bb016dcbca7308fd", size = 15405638, upload-time = "2026-05-11T18:33:48.249Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c2/15c46613b24a84fad2aea1248bf9619b99c2767ae9071fe224c179a0b7d4/mypy-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e79ebc1b904b84f0310dff7469655a9c36c7a68bddb37bdd42b67a332df61d08", size = 16215852, upload-time = "2026-05-11T18:32:50.296Z" }, + { url = "https://files.pythonhosted.org/packages/5c/90/9c16a57f482c76d25f6379762b56bbf65c711d8158cf271fb2802cfb0640/mypy-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e583edc957cfb0deb142079162ae826f58449b116c1d442f2d91c69d9fced081", size = 16452695, upload-time = "2026-05-11T18:33:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4c/215a4eeb63cacc5f17f516691ea7285d11e249802b942476bff15922a314/mypy-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b33b6cd332695bba180d55e717a79d3038e479a2c49cc5eb3d53603409b9a5d7", size = 12866622, upload-time = "2026-05-11T18:34:39.945Z" }, + { url = "https://files.pythonhosted.org/packages/4b/50/1043e1db5f455ffe4c9ab22747cd8ca2bc492b1e4f4e21b130a44ee2b217/mypy-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:4f910fe825376a7b66ef7ca8c98e5a149e8cd64c19ae71d84047a74ee060d4e6", size = 10610798, upload-time = "2026-05-11T18:36:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2a/13ca1f292f6db1b98ff495ef3467736b331621c5917cad984b7043e7348d/mypy-2.1.0-py3-none-any.whl", hash = "sha256:a663814603a5c563fb87a4f96fb473eeb30d1f5a4885afcf44f9db000a366289", size = 2693302, upload-time = "2026-05-11T18:31:29.246Z" }, ] [[package]] @@ -1104,15 +947,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] -[[package]] -name = "narwhals" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/f3/257adc69a71011b4c8cda321b00f02c5bf1980ae38ffd05a58d9632d4de8/narwhals-2.20.0.tar.gz", hash = "sha256:c10994975fa7dc5a68c2cffcddbd5908fc8ebb2d463c5bab085309c0ee1f551e", size = 627848, upload-time = "2026-04-20T12:11:45.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/69/f24d3d1c38ad69e256138b4ec2452a8c7cf66be49dc214771ae99dd4f0a0/narwhals-2.20.0-py3-none-any.whl", hash = "sha256:16e750ea5507d4ba6e8d03455b5f93a535e0405976561baea235bca5dc9f475d", size = 449373, upload-time = "2026-04-20T12:11:43.596Z" }, -] - [[package]] name = "netcdf4" version = "1.7.4" @@ -1140,6 +974,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/2e/39d5e9179c543f2e6e149a65908f83afd9b6d64379a90789b323111761db/netcdf4-1.7.4-cp314-cp314t-win_arm64.whl", hash = "sha256:034220887d48da032cb2db5958f69759dbb04eb33e279ec6390571d4aea734fe", size = 2531682, upload-time = "2026-01-05T02:27:37.062Z" }, ] +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1149,33 +992,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "numcodecs" +version = "0.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/9e/38e7ca8184c958b51f45d56a4aeceb1134ecde2d8bd157efadc98502cc42/numcodecs-0.16.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b05647b8b769e6bc8016e9fd4843c823ce5c9f2337c089fb5c9c4da05e5275de", size = 1654721, upload-time = "2025-11-21T02:49:40.602Z" }, + { url = "https://files.pythonhosted.org/packages/a1/37/260fa42e7b2b08e6e00ad632f8dd620961a60a459426c26cea390f8c68d0/numcodecs-0.16.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3832bd1b5af8bb3e413076b7d93318c8e7d7b68935006b9fa36ca057d1725a8f", size = 1146887, upload-time = "2025-11-21T02:49:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, + { url = "https://files.pythonhosted.org/packages/31/a5/a0425af36c20d55a3ea884db4b4efca25a43bea9214ba69ca7932dd997b4/numcodecs-0.16.5-cp314-cp314-win_amd64.whl", hash = "sha256:b16a14303800e9fb88abc39463ab4706c037647ac17e49e297faa5f7d7dbbf1d", size = 819022, upload-time = "2025-11-21T02:49:47.39Z" }, +] + [[package]] name = "numpy" -version = "2.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/06/c54062f85f673dd5c04cbe2f14c3acb8c8b95e3384869bb8cc9bff8cb9df/numpy-2.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", size = 16684353, upload-time = "2026-03-29T13:20:29.504Z" }, - { url = "https://files.pythonhosted.org/packages/4c/39/8a320264a84404c74cc7e79715de85d6130fa07a0898f67fb5cd5bd79908/numpy-2.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", size = 14704914, upload-time = "2026-03-29T13:20:33.547Z" }, - { url = "https://files.pythonhosted.org/packages/91/fb/287076b2614e1d1044235f50f03748f31fa287e3dbe6abeb35cdfa351eca/numpy-2.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", size = 5210005, upload-time = "2026-03-29T13:20:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/63/eb/fcc338595309910de6ecabfcef2419a9ce24399680bfb149421fa2df1280/numpy-2.4.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", size = 6544974, upload-time = "2026-03-29T13:20:39.014Z" }, - { url = "https://files.pythonhosted.org/packages/44/5d/e7e9044032a716cdfaa3fba27a8e874bf1c5f1912a1ddd4ed071bf8a14a6/numpy-2.4.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", size = 15684591, upload-time = "2026-03-29T13:20:42.146Z" }, - { url = "https://files.pythonhosted.org/packages/98/7c/21252050676612625449b4807d6b695b9ce8a7c9e1c197ee6216c8a65c7c/numpy-2.4.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", size = 16637700, upload-time = "2026-03-29T13:20:46.204Z" }, - { url = "https://files.pythonhosted.org/packages/b1/29/56d2bbef9465db24ef25393383d761a1af4f446a1df9b8cded4fe3a5a5d7/numpy-2.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", size = 17035781, upload-time = "2026-03-29T13:20:50.242Z" }, - { url = "https://files.pythonhosted.org/packages/e3/2b/a35a6d7589d21f44cea7d0a98de5ddcbb3d421b2622a5c96b1edf18707c3/numpy-2.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", size = 18362959, upload-time = "2026-03-29T13:20:54.019Z" }, - { url = "https://files.pythonhosted.org/packages/64/c9/d52ec581f2390e0f5f85cbfd80fb83d965fc15e9f0e1aec2195faa142cde/numpy-2.4.4-cp314-cp314-win32.whl", hash = "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", size = 6008768, upload-time = "2026-03-29T13:20:56.912Z" }, - { url = "https://files.pythonhosted.org/packages/fa/22/4cc31a62a6c7b74a8730e31a4274c5dc80e005751e277a2ce38e675e4923/numpy-2.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", size = 12449181, upload-time = "2026-03-29T13:20:59.548Z" }, - { url = "https://files.pythonhosted.org/packages/70/2e/14cda6f4d8e396c612d1bf97f22958e92148801d7e4f110cabebdc0eef4b/numpy-2.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", size = 10496035, upload-time = "2026-03-29T13:21:02.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e8/8fed8c8d848d7ecea092dc3469643f9d10bc3a134a815a3b033da1d2039b/numpy-2.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", size = 14824958, upload-time = "2026-03-29T13:21:05.671Z" }, - { url = "https://files.pythonhosted.org/packages/05/1a/d8007a5138c179c2bf33ef44503e83d70434d2642877ee8fbb230e7c0548/numpy-2.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", size = 5330020, upload-time = "2026-03-29T13:21:08.635Z" }, - { url = "https://files.pythonhosted.org/packages/99/64/ffb99ac6ae93faf117bcbd5c7ba48a7f45364a33e8e458545d3633615dda/numpy-2.4.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", size = 6650758, upload-time = "2026-03-29T13:21:10.949Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6e/795cc078b78a384052e73b2f6281ff7a700e9bf53bcce2ee579d4f6dd879/numpy-2.4.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", size = 15729948, upload-time = "2026-03-29T13:21:14.047Z" }, - { url = "https://files.pythonhosted.org/packages/5f/86/2acbda8cc2af5f3d7bfc791192863b9e3e19674da7b5e533fded124d1299/numpy-2.4.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", size = 16679325, upload-time = "2026-03-29T13:21:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/bc/59/cafd83018f4aa55e0ac6fa92aa066c0a1877b77a615ceff1711c260ffae8/numpy-2.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", size = 17084883, upload-time = "2026-03-29T13:21:21.106Z" }, - { url = "https://files.pythonhosted.org/packages/f0/85/a42548db84e65ece46ab2caea3d3f78b416a47af387fcbb47ec28e660dc2/numpy-2.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", size = 18403474, upload-time = "2026-03-29T13:21:24.828Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ad/483d9e262f4b831000062e5d8a45e342166ec8aaa1195264982bca267e62/numpy-2.4.4-cp314-cp314t-win32.whl", hash = "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", size = 6155500, upload-time = "2026-03-29T13:21:28.205Z" }, - { url = "https://files.pythonhosted.org/packages/c7/03/2fc4e14c7bd4ff2964b74ba90ecb8552540b6315f201df70f137faa5c589/numpy-2.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", size = 12637755, upload-time = "2026-03-29T13:21:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, ] [[package]] @@ -1198,50 +1058,56 @@ wheels = [ [[package]] name = "pandas" -version = "2.3.3" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, - { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, - { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, - { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, - { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, - { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, - { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, - { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, - { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, ] [[package]] name = "pandas-stubs" -version = "3.0.0.260204" +version = "3.0.3.260530" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/27/1d/297ff2c7ea50a768a2247621d6451abb2a07c0e9be7ca6d36ebe371658e5/pandas_stubs-3.0.0.260204.tar.gz", hash = "sha256:bf9294b76352effcffa9cb85edf0bed1339a7ec0c30b8e1ac3d66b4228f1fbc3", size = 109383, upload-time = "2026-02-04T15:17:17.247Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/aa/c41a8a0ff86fd85dbb3ec0c1f3fa488ca64a8b5f82654ae1b07d84acefe5/pandas_stubs-3.0.3.260530.tar.gz", hash = "sha256:d1efe47b2e5a312c047d7feabec5cb7a55365747983420077e9fcbe9ab74f714", size = 113183, upload-time = "2026-05-30T17:47:40.34Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/2f/f91e4eee21585ff548e83358332d5632ee49f6b2dcd96cb5dca4e0468951/pandas_stubs-3.0.0.260204-py3-none-any.whl", hash = "sha256:5ab9e4d55a6e2752e9720828564af40d48c4f709e6a2c69b743014a6fcb6c241", size = 168540, upload-time = "2026-02-04T15:17:15.615Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e0/99ec5b02203c4e9ce878bc63d8caa06ac1f891e4d63bded9a5ced70fcb4f/pandas_stubs-3.0.3.260530-py3-none-any.whl", hash = "sha256:a6277eb1c8cebf48d9b2413fcd2e9a6b4ff479c934a223c29eacbc3058c4cb55", size = 173780, upload-time = "2026-05-30T17:47:39.13Z" }, ] [[package]] -name = "parso" -version = "0.8.6" +name = "partd" +version = "1.4.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/76/a1e769043c0c0c9fe391b702539d594731a4362334cdf4dc25d0c09761e7/parso-0.8.6.tar.gz", hash = "sha256:2b9a0332696df97d454fa67b81618fd69c35a7b90327cbe6ba5c92d2c68a7bfd", size = 401621, upload-time = "2026-02-09T15:45:24.425Z" } +dependencies = [ + { name = "locket" }, + { name = "toolz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" }, + { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, ] [[package]] @@ -1253,71 +1119,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] -[[package]] -name = "pexpect" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, -] - -[[package]] -name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, - { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, - { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, - { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, - { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, - { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, - { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, - { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, -] - [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, -] - -[[package]] -name = "plotly" -version = "6.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "narwhals" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/7f/0f100df1172aadf88a929a9dbb902656b0880ba4b960fe5224867159d8f4/plotly-6.7.0.tar.gz", hash = "sha256:45eea0ff27e2a23ccd62776f77eb43aa1ca03df4192b76036e380bb479b892c6", size = 6911286, upload-time = "2026-04-09T20:36:45.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/ad/cba91b3bcf04073e4d1655a5c1710ef3f457f56f7d1b79dcc3d72f4dd912/plotly-6.7.0-py3-none-any.whl", hash = "sha256:ac8aca1c25c663a59b5b9140a549264a5badde2e057d79b8c772ae2920e32ff0", size = 9898444, upload-time = "2026-04-09T20:36:39.812Z" }, + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] [[package]] @@ -1331,7 +1139,7 @@ wheels = [ [[package]] name = "pre-commit" -version = "3.8.0" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cfgv" }, @@ -1340,102 +1148,101 @@ dependencies = [ { name = "pyyaml" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/10/97ee2fa54dff1e9da9badbc5e35d0bbaef0776271ea5907eccf64140f72f/pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af", size = 177815, upload-time = "2024-07-28T19:59:01.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/92/caae8c86e94681b42c246f0bca35c059a2f0529e5b92619f6aba4cf7e7b6/pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f", size = 204643, upload-time = "2024-07-28T19:58:59.335Z" }, -] - -[[package]] -name = "prompt-toolkit" -version = "3.0.52" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/2de9408ac81acbb8a7d05d4cc064a152ccf33b3d480ebe0cd292153db239/pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9", size = 198525, upload-time = "2026-04-21T20:31:41.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] name = "proto-plus" -version = "1.27.2" +version = "1.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/0d/94dfe80193e79d55258345901acd2917523d56e8381bc4dee7fd38e3868a/proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24", size = 57204, upload-time = "2026-03-26T22:18:57.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/f3/1fba73eeffafc998a25d59703b63f8be4fe8a5cb12eaff7386a0ba0f7125/proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", size = 50450, upload-time = "2026-03-26T22:13:42.927Z" }, + { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, ] [[package]] name = "protobuf" -version = "6.33.6" +version = "7.35.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, ] [[package]] -name = "psutil" -version = "7.2.2" +name = "pyarrow" +version = "24.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, - { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, - { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, - { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, - { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, -] - -[[package]] -name = "psygnal" -version = "0.15.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/79/20c3e23e75272e9ddf018097cf872ab088bccba978888472656629efa4a3/psygnal-0.15.1.tar.gz", hash = "sha256:f64f62dee2306fc1c22050a59b6c6cdad126e04b0cf50e393ff858a1da719096", size = 123147, upload-time = "2026-01-04T16:38:41.959Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/2e/975bd61727578d88df62797f78390965ca7905780cf01eb59cb095a13638/psygnal-0.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:803fc33c4280c822c6f4b22e6c3ea7c4483e190f3cc69e69350098b3799476f3", size = 595706, upload-time = "2026-01-04T16:38:34.139Z" }, - { url = "https://files.pythonhosted.org/packages/b8/55/e487f1d91497eb75e86c3fdfef69a21b1cab24d023383dd7648b08797d6a/psygnal-0.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4f53b4b83355b0a785b745987fd04e59bbf169a9028ed81a68ca7e05fb76d458", size = 575133, upload-time = "2026-01-04T16:38:35.448Z" }, - { url = "https://files.pythonhosted.org/packages/bf/2f/f286355accd0e68d3eef52e63c8b9ab6ba33ec3107177719a036b3319657/psygnal-0.15.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bcbca12190f5aa65c1f8fb04a81fa6f4463c5f5dde25cd74c3a56ceff6f37b02", size = 889565, upload-time = "2026-01-04T16:38:37.003Z" }, - { url = "https://files.pythonhosted.org/packages/fc/dc/40c6026c88d7f9220ecc913afe0501045a512c9b82f9b7e036bb089dc287/psygnal-0.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1ac399566852fe4354ce26a1acbe12319232e8c2b615fe5ad1e114c547095cf6", size = 880863, upload-time = "2026-01-04T16:38:38.381Z" }, - { url = "https://files.pythonhosted.org/packages/b7/85/b4f45ec3057c473b5622fc002b3a636a698c34d3a0917a064ff5247f1984/psygnal-0.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:d3a03055f331ce91d44581c71edb79938ccc133a94af2ce7ad3a18fa57ac7be5", size = 423654, upload-time = "2026-01-04T16:38:39.7Z" }, - { url = "https://files.pythonhosted.org/packages/46/49/7742544684bee728ec123515d2694cee859aa2a705951a461230b00f18cc/psygnal-0.15.1-py3-none-any.whl", hash = "sha256:4221140e633e45b076953c64bcb9b41a744833527f9a037c1ca98bc270798cbf", size = 90638, upload-time = "2026-01-04T16:38:40.841Z" }, -] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] [[package]] @@ -1468,6 +1275,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -1478,21 +1341,37 @@ wheels = [ ] [[package]] -name = "pyparsing" -version = "3.3.2" +name = "pyogrio" +version = "0.12.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +dependencies = [ + { name = "certifi" }, + { name = "numpy" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/d4/12f86b1ed09721363da4c09622464b604c851a9223fc0c6b393fb2012208/pyogrio-0.12.1.tar.gz", hash = "sha256:e548ab705bb3e5383693717de1e6c76da97f3762ab92522cb310f93128a75ff1", size = 303289, upload-time = "2025-11-28T19:04:53.341Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6f/b4d5e285e08c0c60bcc23b50d73038ddc7335d8de79cc25678cd486a3db0/pyogrio-0.12.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5a1b0453d1c9e7b03715dd57296c8f3790acb8b50d7e3b5844b3074a18f50709", size = 23660673, upload-time = "2025-11-28T19:04:21.662Z" }, + { url = "https://files.pythonhosted.org/packages/8d/75/4b29e71489c5551aa1a1c5ca8c5160a60203c94f2f68c87c0e3614d58965/pyogrio-0.12.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e7ee560422239dd09ca7f8284cc8483a8919c30d25f3049bb0249bff4c38dec4", size = 25232194, upload-time = "2025-11-28T19:04:23.975Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/e9929d2261a07c36301983de2767bcde90d441ab5bf1d767ce56dd07f8b4/pyogrio-0.12.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:648c6f7f5f214d30e6cf493b4af1d59782907ac068af9119ca35f18153d6865a", size = 31336936, upload-time = "2025-11-28T19:04:26.594Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9e/c59941d734ed936d4e5c89b4b99cb5541307cc42b3fd466ee78a1850c177/pyogrio-0.12.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:58042584f3fd4cabb0f55d26c1405053f656be8a5c266c38140316a1e981aca0", size = 30902210, upload-time = "2025-11-28T19:04:29.143Z" }, + { url = "https://files.pythonhosted.org/packages/d1/68/cc07320a63f9c2586e60bf11d148b00e12d0e707673bffe609bbdcb7e754/pyogrio-0.12.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b438e38e4ccbaedaa5cb5824ff5de5539315d9b2fde6547c1e816576924ee8ca", size = 32461674, upload-time = "2025-11-28T19:04:31.792Z" }, + { url = "https://files.pythonhosted.org/packages/13/bc/e4522f429c45a3b6ad28185849dd76e5c8718b780883c4795e7ee41841ae/pyogrio-0.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:f1d8d8a2fea3781dc2a05982c050259261ebc0f6c5e03732d6d79d582adf9363", size = 23550575, upload-time = "2025-11-28T19:04:34.556Z" }, + { url = "https://files.pythonhosted.org/packages/bd/ac/34f0664d0e391994a7b68529ae07a96432b2b4926dbac173ddc4ec94d310/pyogrio-0.12.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:9fe7286946f35a73e6370dc5855bc7a5e8e7babf9e4a8bad7a3279a1d94c7ea9", size = 23694285, upload-time = "2025-11-28T19:04:37.833Z" }, + { url = "https://files.pythonhosted.org/packages/8a/93/873255529faff1da09d0b27287e85ec805a318c60c0c74fd7df77f94e557/pyogrio-0.12.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2c50345b382f1be801d654ec22c70ee974d6057d4ba7afe984b55f2192bc94ee", size = 25259825, upload-time = "2025-11-28T19:04:40.125Z" }, + { url = "https://files.pythonhosted.org/packages/27/95/4d4c3644695d99c6fa0b0b42f0d6266ae9dfaf64478a3371eaac950bdd02/pyogrio-0.12.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0db95765ac0ca935c7fe579e29451294e3ab19c317b0c59c31fbe92a69155e0", size = 31371995, upload-time = "2025-11-28T19:04:42.736Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6f/71f6bcca8754c8bf55a4b7153c61c91f8ac5ba992568e9fa3e54a0ee76fd/pyogrio-0.12.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fc882779075982b93064b3bf3d8642514a6df00d9dd752493b104817072cfb01", size = 31035498, upload-time = "2025-11-28T19:04:45.79Z" }, + { url = "https://files.pythonhosted.org/packages/fd/47/75c1aa165a988347317afab9b938a01ad25dbca559b582ea34473703dc38/pyogrio-0.12.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:806f620e0c54b54dbdd65e9b6368d24f344cda84c9343364b40a57eb3e1c4dca", size = 32496390, upload-time = "2025-11-28T19:04:48.786Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/4641dc5d952f6bdb71dabad2c50e3f8a5d58396cdea6ff8f8a08bfd4f4a6/pyogrio-0.12.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5399f66730978d8852ef5f44dbafa0f738e7f28f4f784349f36830b69a9d2134", size = 23620996, upload-time = "2025-11-28T19:04:51.132Z" }, ] [[package]] -name = "pyperclip" -version = "1.11.0" +name = "pyparsing" +version = "3.3.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] [[package]] @@ -1524,18 +1403,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844, upload-time = "2025-08-14T12:05:40.745Z" }, ] -[[package]] -name = "pyshp" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/20/8b07bae73aaa0c3f5a2683ba6e23b46e977e2d33a88126d56bbcc2d135cd/pyshp-3.0.3.tar.gz", hash = "sha256:bf4678b13dd53578ed87669676a2fffeccbcded1ec8ff9cafb36d1b660f4b305", size = 2192568, upload-time = "2025-11-28T17:47:31.616Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/06/cad54e8ce758bd836ee5411691cbd49efeb9cc611b374670fce299519334/pyshp-3.0.3-py3-none-any.whl", hash = "sha256:28c8fac8c0c25bb0fecbbfd10ead7f319c2ff2f3b0b44a94f22bd2c93510ad42", size = 58465, upload-time = "2025-11-28T17:47:30.328Z" }, -] - [[package]] name = "pytest" -version = "8.4.2" +version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1544,20 +1414,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, -] - -[[package]] -name = "python-box" -version = "7.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/0f/34e7ee0a72f1464b4c7a2e8bafb389f230477256af586bc82bcfad85295a/python_box-7.4.1.tar.gz", hash = "sha256:e412e36c25fca8223560516d53ef6c7993591c3b0ec8bb4ec582bf7defdd79f0", size = 49859, upload-time = "2026-02-21T16:21:16.008Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/a2/771b5e526bba2214ac2d30e321209a66680c40788616a45cf01005e95204/python_box-7.4.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:33c6701faa51fd87f0dcc538873c0fad2b3a1cc3750eab85835cd071cadf1948", size = 1875508, upload-time = "2026-02-21T16:21:37.432Z" }, - { url = "https://files.pythonhosted.org/packages/a6/5f/0e7ea7640ba60ff459ce37e340d816ac5e91b7a9a7c3c161f9dabe622be6/python_box-7.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:ae8c540a0457f52350211d24690211251912018e1e0c1857f50792729d6f562c", size = 1314304, upload-time = "2026-02-21T16:22:22.173Z" }, - { url = "https://files.pythonhosted.org/packages/06/a6/5d3f3abf46b37aa44b1f6788d287c8b4f2319b55013191dddf25b9e6d62c/python_box-7.4.1-py3-none-any.whl", hash = "sha256:a3b0d84d003882fb6abe505b1b883b3a5dcbf226b0fe168d24bc5ff75d9826e5", size = 30402, upload-time = "2026-02-21T16:21:14.78Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, ] [[package]] @@ -1574,24 +1433,24 @@ wheels = [ [[package]] name = "python-discovery" -version = "1.2.2" +version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/ef/3bae0e537cfe91e8431efcba4434463d2c5a65f5a89edd47c6cf2f03c55f/python_discovery-1.2.2.tar.gz", hash = "sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb", size = 58872, upload-time = "2026-04-07T17:28:49.249Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/1a/cbbaf13b730abb0a16b964d984e19f2fe520c21a4dc664051359a3f5a9e7/python_discovery-1.4.2.tar.gz", hash = "sha256:8f3746c4b4968d22afbb97d36e1a0e5b66e6c0f297290f2e95f05b9b8bf18690", size = 70277, upload-time = "2026-06-11T16:10:42.383Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/db/795879cc3ddfe338599bddea6388cc5100b088db0a4caf6e6c1af1c27e04/python_discovery-1.2.2-py3-none-any.whl", hash = "sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a", size = 31894, upload-time = "2026-04-07T17:28:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/1a/82/a70006589557f267f15bd384c0642ad49f0d97b690c3a05b166b9dcbad3b/python_discovery-1.4.2-py3-none-any.whl", hash = "sha256:475803f53b7b2ed6e490e27373f9d8340f7d2eebf9acdaf645d7d714c97bb500", size = 33886, upload-time = "2026-06-11T16:10:41.192Z" }, ] [[package]] -name = "pytz" -version = "2026.1.post1" +name = "python-dotenv" +version = "1.2.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/db/b8721d71d945e6a8ac63c0fc900b2067181dbb50805958d4d4661cf7d277/pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1", size = 321088, upload-time = "2026-03-03T07:47:50.683Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/99/781fe0c827be2742bcc775efefccb3b048a3a9c6ce9aec0cbf4a101677e5/pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a", size = 510489, upload-time = "2026-03-03T07:47:49.167Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] [[package]] @@ -1622,49 +1481,36 @@ wheels = [ [[package]] name = "rasterio" -version = "1.4.4" +version = "1.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "affine" }, { name = "attrs" }, { name = "certifi" }, { name = "click" }, - { name = "click-plugins" }, { name = "cligj" }, { name = "numpy" }, { name = "pyparsing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/fa/fce8dc9f09e5bc6520b6fc1b4ecfa510af9ca06eb42ad7bdff9c9b8989d0/rasterio-1.4.4.tar.gz", hash = "sha256:c95424e2c7f009b8f7df1095d645c52895cd332c0c2e1b4c2e073ea28b930320", size = 445004, upload-time = "2025-12-12T18:01:08.971Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/01/d5a3dc51cd5fef62b76ecc77d33c1ca20de305fed7e16c71bcdf4858e466/rasterio-1.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:019693f14a83ae9225cb57c16e466901d0e6284962dcf13a9f4bb1175b979011", size = 21120237, upload-time = "2025-12-12T18:00:27.723Z" }, - { url = "https://files.pythonhosted.org/packages/50/da/db18362602b17327c0e00c9e9c0847c1c4ac657c1a289169ca06a26faccb/rasterio-1.4.4-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:87d7c3e97e3b40c9041d1602e2dcb4fc2d88abe6c645fccb4939dec297a91cf8", size = 25720506, upload-time = "2025-12-12T18:00:30.592Z" }, - { url = "https://files.pythonhosted.org/packages/5a/8f/a15d66c9c05bffb176c9707ef1f2bfcf9c0b835272937c80ac7207a20b5c/rasterio-1.4.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a2401e4c43a31c7382154d4042b60a63b9bca5886802983c5c9362cdc5b09548", size = 34153931, upload-time = "2025-12-12T18:00:33.852Z" }, - { url = "https://files.pythonhosted.org/packages/05/2d/cd778286b910db7a3f0bc1743ca362173f1fbb7365137e4982ca857b6d26/rasterio-1.4.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6c4287d8934d953f7870b8e2a1df1096fbf47eba39ad0f777a31ea500f4e5010", size = 35421139, upload-time = "2025-12-12T18:00:37.482Z" }, - { url = "https://files.pythonhosted.org/packages/70/97/13a2e33aede8d7a42178c696a6a93868d1f9560f73de05033a1675f0806a/rasterio-1.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:c3ba1871549221140661227dd4fa1f9a472ded4a6d2f2c2e367b0648bb15b99d", size = 26419132, upload-time = "2025-12-12T18:00:40.871Z" }, - { url = "https://files.pythonhosted.org/packages/27/d8/2dcfcb362d6a2fd07c14cfb803a345a7926d4d9fb6243e196df105671e97/rasterio-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:7c9d7dc824cb8d222808be153643cd4e65ea3e1f66019ada1ccd630221edfe30", size = 24800998, upload-time = "2025-12-12T18:00:45.332Z" }, - { url = "https://files.pythonhosted.org/packages/13/f8/16e9b648e7f16cadb41df7c0116dbab26b4a2ba02c85cbe3f744065bdf56/rasterio-1.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:98e17bded830a59992d9f8f8d9f227ce1c4be0694930afcc4360358f5cb1a5db", size = 21247046, upload-time = "2025-12-12T18:00:49.429Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ea/f3dc3a25d7591821d488f5c5eb89f6abcd1f5c8e2ef4bd2792f965cbc9c8/rasterio-1.4.4-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:56134ca203f952855e60774b06672033cf65057eb9810fcc5c1a75f1921053a3", size = 25821677, upload-time = "2025-12-12T18:00:52.458Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d3/1e038350218e852f904c8dc4ab751aa023a2e82e68998767b7b42e33832c/rasterio-1.4.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:52edde65515b33fe4314c8a44a9ee2fc00b550deed6d56e1a8d085d42bbca3e6", size = 34829572, upload-time = "2025-12-12T18:00:56.294Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ce/28abf7a5f5d9cb014c2e14cc396bebe953b3deefbf604d49f4322e73fa35/rasterio-1.4.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d61d3f2c171c64050bd75e54a5d964ff7f165b3f5d2b92c9ee09b9716aa1b8bf", size = 35735171, upload-time = "2025-12-12T18:00:59.531Z" }, - { url = "https://files.pythonhosted.org/packages/54/91/1ce35cfda2d56dacd6395faf20a5290268bd9009c53393ac42b5f9bb2c4c/rasterio-1.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:40137fe512c0d6e96c0167a0ae4e56d82c488f244163c45494b7392e51c844de", size = 26700712, upload-time = "2025-12-12T18:01:03.023Z" }, - { url = "https://files.pythonhosted.org/packages/3b/33/4d13f48a8f01d782ffc1eece20821586518f3f515dca7cf152bca9fd22d4/rasterio-1.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:29ec3a794454b5bb255c9c0374cc380030a8a1e295c81eee7feb036802d2a9e3", size = 24875933, upload-time = "2025-12-12T18:01:06.134Z" }, -] - -[[package]] -name = "ratelim" -version = "0.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "decorator" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/5a/e1440017bccb14523bb76356e6f3a5468386b8a9192bd901e98babd1a1ea/ratelim-0.1.6.tar.gz", hash = "sha256:826d32177e11f9a12831901c9fda6679fd5bbea3605910820167088f5acbb11d", size = 2793, upload-time = "2015-02-27T18:06:08.53Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/88/edb4b66b6cb2c13f123af5a3896bf70c0cbe73ab3cd4243cb4eb0212a0f6/rasterio-1.5.0.tar.gz", hash = "sha256:1e0ea56b02eea4989b36edf8e58a5a3ef40e1b7edcb04def2603accd5ab3ee7b", size = 452184, upload-time = "2026-01-05T16:06:47.169Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl", hash = "sha256:e1a7dd39e6b552b7cc7f52169cd66cdb826a1a30198e355d7016012987c9ad08", size = 4017, upload-time = "2015-02-27T18:06:06.464Z" }, + { url = "https://files.pythonhosted.org/packages/77/9f/f84dfa54110c1c82f9f4fd929465d12519569b6f5d015273aa0957013b2e/rasterio-1.5.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:597be8df418d5ba7b6a927b6b9febfcb42b192882448a8d5b2e2e75a1296631f", size = 22788832, upload-time = "2026-01-05T16:06:12.247Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/de55255c918b17afd7292f793a3500c4aea7e9530b2b3f5b3a57836c7d49/rasterio-1.5.0-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:dd292030d39d685c0b35eddef233e7f1cb8b43052578a3ec97a2da57799693be", size = 24405917, upload-time = "2026-01-05T16:06:14.603Z" }, + { url = "https://files.pythonhosted.org/packages/a9/57/054087a9d5011ad5dfa799277ba8814e41775e1967d37a59ab7b8e2f1876/rasterio-1.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:62c3f97a3c72643c74f2d0f310621a09c35c0c412229c327ae6bcc1ee4b9c3bc", size = 35987536, upload-time = "2026-01-05T16:06:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/c9/72/5fbe5f67ae75d7e89ffb718c500d5fecbaa84f6ba354db306de689faf961/rasterio-1.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:19577f0f0c5f1158af47b57f73356961cbd1782a5f6ae6f3adf6f2650f4eb369", size = 37408048, upload-time = "2026-01-05T16:06:20.82Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3e/0c4ef19980204bdcbc8f9e084056adebc97916ff4edcc718750ef34e5bf9/rasterio-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:015c1ab6e5453312c5e29692752e7ad73568fe4d13567cbd448d7893128cbd2d", size = 30949590, upload-time = "2026-01-05T16:06:23.425Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d8/2e6b81505408926c00e629d7d3d73fd0454213201bd9907450e0fe82f3dd/rasterio-1.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:ff677c0a9d3ba667c067227ef2b76872488b37ff29b061bc3e576fad9baa3286", size = 29337287, upload-time = "2026-01-05T16:06:26.599Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/7b6e6afb28d4e3f69f2229f990ed87dfdc21a3e15ca63b96b2fd9ba17d89/rasterio-1.5.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:508251b9c746d8d008771a30c2160ff321bfc3b41f6a1aa8e8ef1dd4a00d97ba", size = 22926149, upload-time = "2026-01-05T16:06:29.617Z" }, + { url = "https://files.pythonhosted.org/packages/24/30/19345d8bc7d2b96c1172594026b9009702e9ab9f0baf07079d3612aaadae/rasterio-1.5.0-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:742841ed48bc70f6ef517b8fa3521f231780bf408fde0aa6d73770337a36374e", size = 24516040, upload-time = "2026-01-05T16:06:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/9e/43/dc7a4518fa78904bc41952cbf346c3c2a88a20e61b479154058392914c0b/rasterio-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c9a9eee49ce9410c2f352b34c370bb3a96bb518b6a7f97b3a72ee4c835fd4b5c", size = 36589519, upload-time = "2026-01-05T16:06:35.922Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f2/8f706083c6c163054d12c7ed6d5ac4e4ed02252b761288d74e6158871b34/rasterio-1.5.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b9fd87a0b63ab5c6267dfb0bc96f54fdf49d000651b9ee85ed37798141cff046", size = 37714599, upload-time = "2026-01-05T16:06:38.818Z" }, + { url = "https://files.pythonhosted.org/packages/a6/d5/bbca726d5fea5864f7e4bcf3ee893095369e93ad51120495e8c40e2aa1a0/rasterio-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f459db8953ba30ca04fcef2b5e1260eeeff0eae8158bd9c3d6adbe56289765cc", size = 31233931, upload-time = "2026-01-05T16:06:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d1/8b017856e63ccaff3cbd0e82490dbb01363a42f3a462a41b1d8a391e1443/rasterio-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f4b9c2c3b5f10469eb9588f105086e68f0279e62cc9095c4edd245e3f9b88c8a", size = 29418321, upload-time = "2026-01-05T16:06:44.758Z" }, ] [[package]] name = "requests" -version = "2.33.1" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -1672,9 +1518,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -1690,9 +1536,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "rio-cogeo" +version = "7.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "morecantile" }, + { name = "pydantic" }, + { name = "rasterio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/01/3b88154233db4cb29cc54df64006537fadb8d7c2ace872af985dac972fd3/rio_cogeo-7.0.2.tar.gz", hash = "sha256:1454ff94dca8652db68862d667bf47e54ecabfe8872c939f2c16ebb62d432f69", size = 19282, upload-time = "2026-03-27T08:25:27.554Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/da/835b573b1a8ba0fcdb3fa6c26fee0c8fa7549f79cf882894ea2238f19f1c/rio_cogeo-7.0.2-py3-none-any.whl", hash = "sha256:1426faea08ae0970707d3534119164d99969ee15181b06d94c5f84645f82537f", size = 21827, upload-time = "2026-03-27T08:25:26.18Z" }, +] + [[package]] name = "rioxarray" -version = "0.19.0" +version = "0.22.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -1701,18 +1562,36 @@ dependencies = [ { name = "rasterio" }, { name = "xarray" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/8e/fe4e87460f8c62d8d5c683e09f19fbde5d9cfcfd0342d02df1f452999b5d/rioxarray-0.19.0.tar.gz", hash = "sha256:7819a0036fd874c8c8e280447cbbe43d8dc72fc4a14ac7852a665b1bdb7d4b04", size = 54600, upload-time = "2025-04-21T17:46:54.183Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/04/9e43477ab0fce7c4c949e1131bfae55ec5228da4ba30f55760660db224b2/rioxarray-0.22.0.tar.gz", hash = "sha256:3f55f23a632ffd9eff13463634227f4afbbcf298947536e161f6cf2ce88d4373", size = 61337, upload-time = "2026-03-06T17:11:00.16Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2f/63d2cacc0e525f8e3398bcf32bd3620385f22cd1600834ec49d7f3597a7b/rioxarray-0.19.0-py3-none-any.whl", hash = "sha256:494ee4fff1781072d55ee5276f5d07b63d93b05093cb33b926a12186ba5bb8ef", size = 62151, upload-time = "2025-04-21T17:46:52.801Z" }, + { url = "https://files.pythonhosted.org/packages/3f/dd/0b2c68495331ba36af783139baaa94693ef310d484d458c11dfa1357287d/rioxarray-0.22.0-py3-none-any.whl", hash = "sha256:db0aa55cd36a95060968f2e6574107829def29d43a563560b90bc642d0bd6a3b", size = 72018, upload-time = "2026-03-06T17:10:58.965Z" }, ] [[package]] -name = "scooby" -version = "0.11.2" +name = "shapely" +version = "2.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/06/9a8600207fd72a29ee965e9a4c61b750cc3fa106768f14a7b3ee3e36cb61/scooby-0.11.2.tar.gz", hash = "sha256:0575c73636ec4c2587bea1f8a038798ddcb249e02067fae897dac3bf4f4e444d", size = 242928, upload-time = "2026-04-22T23:13:12.307Z" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/bc/1173f502f1870e3bae81c148326c5cbcc19ec77df79a9aaf17a59911355c/scooby-0.11.2-py3-none-any.whl", hash = "sha256:f34c36bbee749b2c55816a080521f216d88304e635017e911c12249607d38c49", size = 20142, upload-time = "2026-04-22T23:13:10.705Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:9a522f460d28e2bf4e12396240a5fc1518788b2fcd73535166d748399ef0c223", size = 1831290, upload-time = "2025-09-24T13:51:13.56Z" }, + { url = "https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ff629e00818033b8d71139565527ced7d776c269a49bd78c9df84e8f852190c", size = 1641463, upload-time = "2025-09-24T13:51:14.972Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/91d59ae525ca641e7ac5551c04c9503aee6f29b92b392f31790fcb1a4358/shapely-2.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f67b34271dedc3c653eba4e3d7111aa421d5be9b4c4c7d38d30907f796cb30df", size = 2970145, upload-time = "2025-09-24T13:51:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21952dc00df38a2c28375659b07a3979d22641aeb104751e769c3ee825aadecf", size = 3073806, upload-time = "2025-09-24T13:51:18.712Z" }, + { url = "https://files.pythonhosted.org/packages/03/83/f768a54af775eb41ef2e7bec8a0a0dbe7d2431c3e78c0a8bdba7ab17e446/shapely-2.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1f2f33f486777456586948e333a56ae21f35ae273be99255a191f5c1fa302eb4", size = 3980803, upload-time = "2025-09-24T13:51:20.37Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/559c7c195807c91c79d38a1f6901384a2878a76fbdf3f1048893a9b7534d/shapely-2.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cf831a13e0d5a7eb519e96f58ec26e049b1fad411fc6fc23b162a7ce04d9cffc", size = 4133301, upload-time = "2025-09-24T13:51:21.887Z" }, + { url = "https://files.pythonhosted.org/packages/80/cd/60d5ae203241c53ef3abd2ef27c6800e21afd6c94e39db5315ea0cbafb4a/shapely-2.1.2-cp314-cp314-win32.whl", hash = "sha256:61edcd8d0d17dd99075d320a1dd39c0cb9616f7572f10ef91b4b5b00c4aeb566", size = 1583247, upload-time = "2025-09-24T13:51:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:a444e7afccdb0999e203b976adb37ea633725333e5b119ad40b1ca291ecf311c", size = 1773019, upload-time = "2025-09-24T13:51:24.873Z" }, + { url = "https://files.pythonhosted.org/packages/a3/05/a44f3f9f695fa3ada22786dc9da33c933da1cbc4bfe876fe3a100bafe263/shapely-2.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:5ebe3f84c6112ad3d4632b1fd2290665aa75d4cef5f6c5d77c4c95b324527c6a", size = 1834137, upload-time = "2025-09-24T13:51:26.665Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/4d57db45bf314573427b0a70dfca15d912d108e6023f623947fa69f39b72/shapely-2.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5860eb9f00a1d49ebb14e881f5caf6c2cf472c7fd38bd7f253bbd34f934eb076", size = 1642884, upload-time = "2025-09-24T13:51:28.029Z" }, + { url = "https://files.pythonhosted.org/packages/5a/27/4e29c0a55d6d14ad7422bf86995d7ff3f54af0eba59617eb95caf84b9680/shapely-2.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b705c99c76695702656327b819c9660768ec33f5ce01fa32b2af62b56ba400a1", size = 3018320, upload-time = "2025-09-24T13:51:29.903Z" }, + { url = "https://files.pythonhosted.org/packages/9f/bb/992e6a3c463f4d29d4cd6ab8963b75b1b1040199edbd72beada4af46bde5/shapely-2.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a1fd0ea855b2cf7c9cddaf25543e914dd75af9de08785f20ca3085f2c9ca60b0", size = 3094931, upload-time = "2025-09-24T13:51:32.699Z" }, + { url = "https://files.pythonhosted.org/packages/9c/16/82e65e21070e473f0ed6451224ed9fa0be85033d17e0c6e7213a12f59d12/shapely-2.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:df90e2db118c3671a0754f38e36802db75fe0920d211a27481daf50a711fdf26", size = 4030406, upload-time = "2025-09-24T13:51:34.189Z" }, + { url = "https://files.pythonhosted.org/packages/7c/75/c24ed871c576d7e2b64b04b1fe3d075157f6eb54e59670d3f5ffb36e25c7/shapely-2.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:361b6d45030b4ac64ddd0a26046906c8202eb60d0f9f53085f5179f1d23021a0", size = 4169511, upload-time = "2025-09-24T13:51:36.297Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f7/b3d1d6d18ebf55236eec1c681ce5e665742aab3c0b7b232720a7d43df7b6/shapely-2.1.2-cp314-cp314t-win32.whl", hash = "sha256:b54df60f1fbdecc8ebc2c5b11870461a6417b3d617f555e5033f1505d36e5735", size = 1602607, upload-time = "2025-09-24T13:51:37.757Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/f09272a71976dfc138129b8faf435d064a811ae2f708cb147dccdf7aacdb/shapely-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:0036ac886e0923417932c2e6369b6c52e38e0ff5d9120b90eef5cd9a5fc5cae9", size = 1796682, upload-time = "2025-09-24T13:51:39.233Z" }, ] [[package]] @@ -1725,50 +1604,72 @@ wheels = [ ] [[package]] -name = "stack-data" -version = "0.6.3" +name = "smmap" +version = "5.0.3" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asttokens" }, - { name = "executing" }, - { name = "pure-eval" }, +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } + +[[package]] +name = "toolz" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, + { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, ] [[package]] -name = "traitlets" -version = "5.14.3" +name = "types-geopandas" +version = "1.1.3.20260518" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } +dependencies = [ + { name = "numpy" }, + { name = "pandas-stubs" }, + { name = "pyproj" }, + { name = "types-shapely" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/77/cad393d5276892acebb7276ae839f4dd74bc6aae7794dd9d774c3f1d594d/types_geopandas-1.1.3.20260518.tar.gz", hash = "sha256:ba519db699a484aea32ebb8e6536ced1f6d1caa3c8fdf80f92f8cbb2ddb31d05", size = 23813, upload-time = "2026-05-18T06:07:45.247Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, + { url = "https://files.pythonhosted.org/packages/2b/4a/e71a0f84baec24a4868f7472031bef5250a32d9338ec8285de1fa2b9214e/types_geopandas-1.1.3.20260518-py3-none-any.whl", hash = "sha256:5b55a4e8aa898f33eb0f464fd78dd332f7773ec89c9e4254510fe988757c3b10", size = 31797, upload-time = "2026-05-18T06:07:44.394Z" }, ] [[package]] -name = "traittypes" -version = "0.2.3" +name = "types-networkx" +version = "3.6.1.20260612" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets" }, + { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d6/8d/37d686f52dfbccc47b857751531ffdec262b0f35158dd3b306030dafdb83/traittypes-0.2.3.tar.gz", hash = "sha256:212feed38d566d772648768b78d3347c148ef23915b91c02078188e631316c86", size = 16003, upload-time = "2025-10-22T11:06:09.952Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/d6/86483526df18c9936cb321b33f6df929db20414378f536972c5e9ee20789/types_networkx-3.6.1.20260612.tar.gz", hash = "sha256:9a28345fb3ad985e460667ef2b9c0879920761c8000cd07b424c0bf1b24ca778", size = 73974, upload-time = "2026-06-12T06:22:35.097Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/c0/fdf9d3ee103ce66a55f0532835ad5e154226c5222423c6636ba049dc42fc/traittypes-0.2.3-py2.py3-none-any.whl", hash = "sha256:49016082ce740d6556d9bb4672ee2d899cd14f9365f17cbb79d5d96b47096d4e", size = 8130, upload-time = "2025-10-22T11:06:08.824Z" }, + { url = "https://files.pythonhosted.org/packages/6e/1d/4694309c774f552a0dad8d357bc8c2b152120c9744dad4f21fdc3e819c37/types_networkx-3.6.1.20260612-py3-none-any.whl", hash = "sha256:829cdad128c10e072aacadd0421d7ebd64cee9731110e9d6b3f22b575fe0bbfd", size = 162459, upload-time = "2026-06-12T06:22:33.875Z" }, ] [[package]] name = "types-requests" -version = "2.33.0.20260508" +version = "2.33.0.20260518" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/6b/eb226bdd61a982c9a03e02c657fb4ab001733506e6423906ac142331f2e3/types_requests-2.33.0.20260508.tar.gz", hash = "sha256:81b2ae5f0d20967714a6aa5ef9284c05570d7cb06b7de8f2a77b918b63ddd411", size = 23991, upload-time = "2026-05-08T04:50:56.818Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/01/c5a19253fe1ac159159ddf9a3a07cec8bb5e486ec4d9002ad2821da0e5d2/types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e", size = 24752, upload-time = "2026-05-18T06:07:37.966Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/96/080db0afdf2c5cc5fe512b41354e8d114fe8f65e9510c56ff8dfd40216ce/types_requests-2.33.0.20260508-py3-none-any.whl", hash = "sha256:fa01459cca184229713df03709db46a905325906d27e042cd4fd7ea3d15d3400", size = 20722, upload-time = "2026-05-08T04:50:55.548Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/b139710a3b6018f7fb2b9508b35c8af564e61bf2bf4fa619d088f3e16f85/types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0", size = 21391, upload-time = "2026-05-18T06:07:37.044Z" }, +] + +[[package]] +name = "types-shapely" +version = "2.1.0.20260603" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/8d/b1ee75d100855ce214788b2e05b3648db50d6f0ff110fa986328a25e1fa1/types_shapely-2.1.0.20260603.tar.gz", hash = "sha256:0373ef71caa8c17aff0b810b8155f744ceaeee802f6f1438d09dd1c919612180", size = 27246, upload-time = "2026-06-03T06:42:05.798Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/e8/dfb9d2f832260d64d3b0b603ac6382ccc9b1a332535e434e9d5f97f7aed3/types_shapely-2.1.0.20260603-py3-none-any.whl", hash = "sha256:bc9ea51658b05db3b92e453cc74614af1d70eb391e4c36e15d80567fea6a61a8", size = 38338, upload-time = "2026-06-03T06:42:04.732Z" }, ] [[package]] @@ -1781,35 +1682,38 @@ wheels = [ ] [[package]] -name = "tzdata" -version = "2026.2" +name = "typing-inspection" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] -name = "uritemplate" -version = "4.2.0" +name = "tzdata" +version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] name = "virtualenv" -version = "21.3.0" +version = "21.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -1817,27 +1721,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/8b/6331f7a7fe70131c301106ec1e7cf23e2501bf7d4ca3636805801ca191bb/virtualenv-21.3.0.tar.gz", hash = "sha256:733750db978ec95c2d8eb4feadaa57091002bce404cb39ba69899cf7bd28944e", size = 7614069, upload-time = "2026-04-27T17:05:58.927Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/eb/03bfb1299d4c4510329e470f13f9a4ce793df7fcb5a2fd3510f911066f61/virtualenv-21.3.0-py3-none-any.whl", hash = "sha256:4d28ee41f6d9ec8f1f00cd472b9ffbcedda1b3d3b9a575b5c94a2d004fd51bd7", size = 7594690, upload-time = "2026-04-27T17:05:55.468Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, -] - -[[package]] -name = "widgetsnbextension" -version = "4.0.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/c67440c7fb409a71b7404b7aefcd7569a9c0d6bd071299bf4198ae7a5d95/widgetsnbextension-4.0.15.tar.gz", hash = "sha256:de8610639996f1567952d763a5a41af8af37f2575a41f9852a38f947eb82a3b9", size = 1097402, upload-time = "2025-11-01T21:15:55.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/0e/933bacb37b57ae7928b0030eef205a3dbb3e37afdbdde5be2e113318958f/virtualenv-21.5.0.tar.gz", hash = "sha256:98847aadf5e2037e0e4d2e19528eb3aca6f23906422e59a510bff231a6d32fce", size = 4577424, upload-time = "2026-06-13T20:36:45.066Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, + { url = "https://files.pythonhosted.org/packages/e9/87/b0667ede418386ab631e48924b845d326f366d61e6bd08fe68a748fae4d4/virtualenv-21.5.0-py3-none-any.whl", hash = "sha256:8f7c38605023688c89789f566959006af6d61c99eeeb9e58342eb780c5761e5e", size = 4557937, upload-time = "2026-06-13T20:36:42.967Z" }, ] [[package]] @@ -1855,10 +1741,66 @@ wheels = [ ] [[package]] -name = "xyzservices" -version = "2026.3.0" +name = "yarl" +version = "1.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670, upload-time = "2026-05-19T21:29:56.631Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916, upload-time = "2026-05-19T21:29:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625, upload-time = "2026-05-19T21:30:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574, upload-time = "2026-05-19T21:30:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534, upload-time = "2026-05-19T21:30:04.319Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481, upload-time = "2026-05-19T21:30:05.988Z" }, + { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529, upload-time = "2026-05-19T21:30:07.738Z" }, + { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338, upload-time = "2026-05-19T21:30:09.713Z" }, + { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147, upload-time = "2026-05-19T21:30:11.365Z" }, + { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272, upload-time = "2026-05-19T21:30:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962, upload-time = "2026-05-19T21:30:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063, upload-time = "2026-05-19T21:30:16.683Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438, upload-time = "2026-05-19T21:30:18.769Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458, upload-time = "2026-05-19T21:30:21.024Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589, upload-time = "2026-05-19T21:30:23.412Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424, upload-time = "2026-05-19T21:30:25.425Z" }, + { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690, upload-time = "2026-05-19T21:30:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248, upload-time = "2026-05-19T21:30:29.297Z" }, + { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084, upload-time = "2026-05-19T21:30:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272, upload-time = "2026-05-19T21:30:33.062Z" }, + { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497, upload-time = "2026-05-19T21:30:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002, upload-time = "2026-05-19T21:30:37.724Z" }, + { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524, upload-time = "2026-05-19T21:30:40.196Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165, upload-time = "2026-05-19T21:30:41.888Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010, upload-time = "2026-05-19T21:30:43.985Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128, upload-time = "2026-05-19T21:30:46.291Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382, upload-time = "2026-05-19T21:30:48.085Z" }, + { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964, upload-time = "2026-05-19T21:30:49.785Z" }, + { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204, upload-time = "2026-05-19T21:30:51.862Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510, upload-time = "2026-05-19T21:30:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584, upload-time = "2026-05-19T21:30:55.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410, upload-time = "2026-05-19T21:30:57.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980, upload-time = "2026-05-19T21:30:59.735Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219, upload-time = "2026-05-19T21:31:01.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, +] + +[[package]] +name = "zarr" +version = "3.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/08/3cb9f67a8d48021aca2a02292cc26eecd71d949ae70ad66420a8730cc302/xyzservices-2026.3.0.tar.gz", hash = "sha256:d226866a5d8e9fef337034d8da37a8298f0a1d9d1489b4018e69579eb321fea4", size = 1135736, upload-time = "2026-03-30T14:42:25.596Z" } +dependencies = [ + { name = "donfig" }, + { name = "google-crc32c" }, + { name = "numcodecs" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/8d/aeb164004f87543b06ef54f885d02c342c31ceb274e2bbec470a98927621/zarr-3.2.1.tar.gz", hash = "sha256:71565b738a0e7e8ed226f0516eba8c6bb53440ad7669a8c48ebb3534a161d035", size = 675161, upload-time = "2026-05-05T12:37:22.383Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a9/d23012099dc88ec69a29c6407b41d89681cb674c2043cd5b467c7e299c08/xyzservices-2026.3.0-py3-none-any.whl", hash = "sha256:503183d4b322bfebc3c50cdd21192aa3e81e36c5efbf9133d54ae82143e0576b", size = 94101, upload-time = "2026-03-30T14:42:24.608Z" }, + { url = "https://files.pythonhosted.org/packages/88/0a/469e2bd01be1490336e6c8707386845655d59261543315778a3ccc7e8019/zarr-3.2.1-py3-none-any.whl", hash = "sha256:f78cdd3d9687ad0e9f9cba2c5683b64f0c52589c19f685eeabe872e93cc0d2c7", size = 319617, upload-time = "2026-05-05T12:37:20.66Z" }, ]