diff --git a/CHANGES.md b/CHANGES.md index f96f9139..49fb2d31 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,10 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin * perf: use `ST_EstimatedExtent` for spatial extents whenever table statistics exist. * perf: cache `Collection.extent`, `bounds`, `dt_bounds`, `geometry_columns` and `datetime_columns` (`functools.cached_property`) instead of recomputing them on every access * perf: skip SQL statement formatting in `tipg.pg.collections.debug_query` when `DEBUG` logging is disabled +* feat: add DuckDB backend (`tipg.duck`, `tipg[duckdb]` extra) to serve DuckDB tables, views and files (GeoParquet, `ST_Read` formats) as OGC Features and Tiles collections, alone or mixed with PostgreSQL collections in the same catalog + * `connect_to_duckdb` / `close_duckdb_connection` to manage the connection (`app.state.duckdb_connection`) + * `duck_get_collection_index(*, connection, settings=None)` to list the collections + * `DuckDBSettings` / `DuckDBCatalogSettings` (`TIPG_DUCKDB_` env prefix) * change: add `Collection._tag_geometry_literal` and `Collection._reproject_geometry_literal` abstract methods, defining how CQL2/bbox geometry literals are tagged with, or reprojected to, the geometry column CRS for each backend **breaking change** (custom `Collection` subclasses must implement them) * change: move `_quote_ident` from `tipg.pg.collections` to `tipg.collections` * docs: update the `examples/non_app_usage.ipynb` notebook for the new `tipg.pg` API diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4e8f6465..8f7cdfa5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -27,11 +27,16 @@ nav: - "Customization": advanced/customization.md - "TiPg Vector Tiles server": advanced/ogc_tiles_server.md - "TiPg Features server": advanced/ogc_features_server.md + - "DuckDB backend": advanced/duckdb.md - API: - pg: - collections: api/tipg/pg/collections.md - database: api/tipg/pg/database.md - settings: api/tipg/pg/settings.md + - duck: + - collections: api/tipg/duck/collections.md + - database: api/tipg/duck/database.md + - settings: api/tipg/duck/settings.md - collections: api/tipg/collections.md - dependencies: api/tipg/dependencies.md - errors: api/tipg/errors.md diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md new file mode 100644 index 00000000..c90b53a5 --- /dev/null +++ b/docs/src/advanced/duckdb.md @@ -0,0 +1,284 @@ + +`TiPg` can serve tables, views and files (GeoParquet, or any format supported by DuckDB's `ST_Read`) through [DuckDB](https://duckdb.org) and its [spatial extension](https://duckdb.org/docs/extensions/spatial/overview), alongside or instead of PostgreSQL/PostGIS. + +!!! warning "Experimental" + + The DuckDB backend is **experimental**: its API, settings and behavior may change in future releases, without following the deprecation policy of the rest of `TiPg`. Please report any issue or feedback on [GitHub](https://github.com/developmentseed/tipg/issues). + +!!! note + + The DuckDB backend is not used by the default `tipg.main:app` application: you need to create your own application (see examples below, and the [example application](#example-application)). + +### Install + +```bash +python -m pip install "tipg[duckdb]" +``` + +The DuckDB backend requires `duckdb>=2.0` (CRS aware `GEOMETRY` type). + +### Settings + +`tipg.duck.DuckDBSettings` (connection) and `tipg.duck.DuckDBCatalogSettings` (catalog) can be set with `TIPG_DUCKDB_` prefixed environment variables. + +| Setting | Env | Default | Description | +| ------- | --- | ------- | ----------- | +| `database` | `TIPG_DUCKDB_DATABASE` | | Path or URL to a DuckDB database file, attached (read-only) under its file name (without extension). If not set, only `sources` are served | +| `threads` | `TIPG_DUCKDB_THREADS` | | Number of threads DuckDB can use | +| `memory_limit` | `TIPG_DUCKDB_MEMORY_LIMIT` | | Maximum memory DuckDB can use (e.g `4GB`) | +| `sources` | `TIPG_DUCKDB_SOURCES` | `{}` | Mapping of name to file path/glob/URL (e.g `'{"countries": "data/countries.parquet"}'`). Parquet files are read with `read_parquet`, other formats with `ST_Read` | +| `schemas` | `TIPG_DUCKDB_SCHEMAS` | `["main"]` | Schemas to look for tables/views in | +| `tables` | `TIPG_DUCKDB_TABLES` | | Only expose those tables (`schema.table`) | +| `exclude_tables` | `TIPG_DUCKDB_EXCLUDE_TABLES` | | Tables (`schema.table`) to exclude | +| `only_spatial_tables` | `TIPG_DUCKDB_ONLY_SPATIAL_TABLES` | `True` | Only expose tables/views with a geometry column | +| `spatial_extent` | `TIPG_DUCKDB_SPATIAL_EXTENT` | `True` | Read the spatial extent and geometry type of parquet sources from their GeoParquet metadata (file footers only). For a glob, every matched file's footer is read on each catalog refresh: disable it for sources matching many remote files | +| `metadata_threads` | `TIPG_DUCKDB_METADATA_THREADS` | `16` | Number of parquet sources whose metadata is read concurrently when building the catalog | + +Table level configuration (`TIPG_TABLE_CONFIG__...`, fallback key names, column sorting) is shared with the PostgreSQL backend (see [Configuration](../user_guide/configuration.md)). + +Collections are identified by `{schema}.{table}`; file `sources` are registered as views in the in-memory `main` schema, so a `countries` source is exposed as the `main.countries` collection. + +### DuckDB application + +```python +"""TiPg DuckDB application.""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + DuckDBSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.factory import Endpoints +from tipg.middleware import CatalogUpdateMiddleware + +duckdb_settings = DuckDBSettings() +catalog_settings = DuckDBCatalogSettings() + + +async def register_collection_catalog(app: FastAPI) -> None: + """Register Collection catalog.""" + collections = await duck_get_collection_index( + connection=app.state.duckdb_connection, + settings=catalog_settings, + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in collections}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + await connect_to_duckdb(app, settings=duckdb_settings) + await register_collection_catalog(app) + yield + await close_duckdb_connection(app) + + +app = FastAPI(title="TiPg DuckDB", lifespan=lifespan) + +ogc_api = Endpoints(title="TiPg DuckDB") +app.include_router(ogc_api.router) + +app.add_middleware(CatalogUpdateMiddleware, func=register_collection_catalog, ttl=300) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) +``` + +```bash +TIPG_DUCKDB_DATABASE=data.duckdb \ +TIPG_DUCKDB_SOURCES='{"countries": "countries.parquet"}' \ +uvicorn app:app --port 8000 +``` + +### Example application + +A complete DuckDB application can be found in [`examples/duckdb`](https://github.com/developmentseed/tipg/tree/main/examples/duckdb): + +- [`main.py`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/main.py): a DuckDB version of `tipg.main:app` (CORS, cache-control and compression middlewares, viewer extension, `/healthz` endpoint), configured with the `TIPG_` and `TIPG_DUCKDB_` environment variables. +- [`build_source_coop_db.py`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/build_source_coop_db.py): a script creating a DuckDB database with one view per [Source Cooperative](https://source.coop) GeoParquet file listed in [`source_coop_geoparquet.md`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/source_coop_geoparquet.md). Views read the remote files directly (no data is copied). +- `source_coop.duckdb`: the database created by the script. + +```bash +cd examples/duckdb + +# Serve a remote GeoParquet file +TIPG_DUCKDB_SOURCES='{"buildings": "https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' \ +uv run --extra duckdb --group server uvicorn main:app + +# Serve the Source Cooperative GeoParquet files +TIPG_DUCKDB_DATABASE=source_coop.duckdb uv run --extra duckdb --group server uvicorn main:app +``` + +### PostgreSQL and DuckDB in a single application + +Each collection carries its own connection (`PgCollection.connection_pool`, `DuckCollection.connection`), so collections from both backends can be served by the same endpoints. Merge the collections from both backends in your `register_collection_catalog` function: + +```python +"""TiPg PostgreSQL + DuckDB application.""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from tipg import duck, pg +from tipg.collections import Catalog +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.factory import Endpoints + +pg_catalog_settings = pg.PostgresCatalogSettings() + + +async def register_collection_catalog(app: FastAPI) -> None: + """Register Collection catalog.""" + pg_collections = await pg.pg_get_collection_index( + connection_pool=app.state.pool, + settings=pg_catalog_settings, + ) + + duck_collections = await duck.duck_get_collection_index( + connection=app.state.duckdb_connection, + ) + + # NOTE: collections are identified by `{schema}.{table}`, make sure + # both backends don't expose the same ids. + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*pg_collections, *duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + await pg.connect_to_db( + app, + schemas=pg_catalog_settings.schemas, + tipg_schema=pg_catalog_settings.tipg_schema, + settings=pg.PostgresSettings(), + ) + await duck.connect_to_duckdb(app, settings=duck.DuckDBSettings()) + await register_collection_catalog(app) + yield + await duck.close_duckdb_connection(app) + await pg.close_db_connection(app) + + +app = FastAPI(title="TiPg", lifespan=lifespan) +app.include_router(Endpoints(title="TiPg").router) +add_exception_handlers(app, DEFAULT_STATUS_CODES) +``` + +### Differences with the PostgreSQL backend + +- **CRS**: the CRS of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. CRS without EPSG code (e.g PROJJSON from GeoParquet metadata, or `IGNF:LAMB93`) are supported: the CRS definition is used as is for reprojection (`Column.srid` is then a string), and the collection `crs` is the OGC URI of the CRS authority code (e.g `http://www.opengis.net/def/crs/IGNF/0/LAMB93`), or `None` when the CRS has no authority code. PROJJSON definitions of EPSG CRS are matched to their EPSG code. +- **Extents and geometry type**: tables and views are never scanned to build the catalog. The spatial extent and geometry type of parquet `sources` are read from their [GeoParquet](https://geoparquet.org) metadata (`bbox` and `geometry_types`, in the file footers). Other collections have no spatial extent and a generic `Geometry` type, and no collection has a temporal extent. +- **Items order**: without `sortby`, features are returned in their insertion order (no default sort on the primary key). +- **Feature ids**: single parquet file `sources` are read with `read_parquet(..., file_row_number = true)`, and the added `file_row_number` column (row position in the file, starting at 0) is used as id when the collection has no other id column (`TIPG_TABLE_CONFIG__..._pk` or fallback key names). Looking up a feature by id (`/items/{itemId}`) then only reads one row group. Other collections without primary key use the feature position in the response as `id`, and don't support `/items/{itemId}`. +- **Spatial filters**: DuckDB has no spatial index on parquet files. When a GeoParquet (1.1+) source declares a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column, spatial filters (`bbox`, tile envelope, and cql2 spatial predicates in the top-level `AND` of the filter) also filter the covering columns, so DuckDB can skip the row groups outside of the filter using their statistics. Without covering, spatial filters scan the whole geometry column. Parquet sources whose covering doesn't match the file (e.g. pointing to missing struct fields) are not added to the catalog (a warning is logged). See [Data layout and performance](#data-layout-and-performance). +- **Parquet metadata cache**: parquet files metadata (footers) are cached in memory (`parquet_metadata_cache`), so they are only fetched once per file. +- **`numberMatched`**: only returned for requests without filter (DuckDB answers `COUNT(*)` from metadata), counting the features matching a filter would need a full scan. +- **Vector tiles**: `ST_AsMVT` only supports boolean, integer, float and string properties; `DECIMAL` values are cast to `DOUBLE` and other types (dates, timestamps, lists, ...) to `VARCHAR`. +- **Functions**: SQL function layers are not supported. +- **WKT output** (CSV, (nd)json): DuckDB's `ST_AsText` formatting (e.g `SRID=4326;POINT (1 2)`). + +### Data layout and performance + +DuckDB has no spatial index: for a `bbox` filter or a tile, it can only skip the data whose row group statistics show that it's outside of the requested area. How the data is laid out (sorted, grouped, with or without bbox columns) has a much bigger impact on performance than anything in `TiPg`'s configuration, especially for remote files where every row group read is an HTTP request. + +For example, a 2.3M polygons GeoParquet file (~870 MB) served from a remote URL: + +| Layout | Data read for one tile | Time | +| ------ | ---------------------- | ---- | +| Rows not sorted (every row group covers the whole dataset extent) | ~870 MB (whole file, 463 HTTP requests) | ~16s | +| Bbox filter on row groups which don't overlap the tile | ~0.6 MB (file footer only, 2 HTTP requests) | ~2s | + +Row groups can only be skipped when: + +1. **the rows are spatially sorted**: features close to each other are stored in the same row groups, so each row group covers a small area. With unsorted rows, every row group covers (almost) the whole dataset extent and overlaps every tile. +2. **DuckDB has statistics to compare with the filter**: a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column for GeoParquet files, or the geometry column statistics for DuckDB tables. + +#### Recommended: DuckDB tables + +For data served often (or from slow remote storage), copy it into a DuckDB database file, sorted along a [Hilbert curve](https://en.wikipedia.org/wiki/Hilbert_curve). DuckDB keeps bounding box statistics of `GEOMETRY` columns for each row group of a table, so no bbox column is needed: + +```sql +INSTALL spatial; +LOAD spatial; + +CREATE TABLE parcels_raw AS SELECT * FROM read_parquet('https://example.com/parcels.parquet'); + +CREATE TABLE parcels AS +SELECT * FROM parcels_raw +ORDER BY ST_Hilbert(geometry, (SELECT ST_Extent(ST_Extent_Agg(geometry)) FROM parcels_raw)); + +DROP TABLE parcels_raw; +``` + +```bash +TIPG_DUCKDB_DATABASE=parcels.duckdb uvicorn app:app --port 8000 +``` + +!!! note + + Tables are not scanned to build the catalog: collections from DuckDB tables have no spatial extent (see [Differences with the PostgreSQL backend](#differences-with-the-postgresql-backend)). + +#### GeoParquet files + +For parquet `sources`, write [GeoParquet 1.1](https://geoparquet.org/releases/v1.1.0/) files with: + +- **spatially sorted rows** (Hilbert curve, or sorted by bounding box) +- **a bbox covering column**, which follows the specification (a struct column with `xmin`, `ymin`, `xmax` and `ymax` fields, declared in the `covering` of the `geo` metadata). Sources with a covering which doesn't match the file are not served. +- **row groups of 50 000 to 100 000 rows**: smaller row groups mean more HTTP requests, bigger ones more data read outside of the requested area. + +With DuckDB (Hilbert sort) and [GDAL](https://gdal.org/en/stable/drivers/vector/parquet.html) (>= 3.9, bbox covering): + +```sql +-- DuckDB +COPY ( + SELECT * FROM 'parcels.parquet' + ORDER BY ST_Hilbert(geometry, (SELECT ST_Extent(ST_Extent_Agg(geometry)) FROM 'parcels.parquet')) +) TO 'parcels_hilbert.parquet' (FORMAT parquet); +``` + +```bash +# GDAL keeps the rows order +ogr2ogr -f Parquet parcels_sorted.parquet parcels_hilbert.parquet \ + -lco WRITE_COVERING_BBOX=YES \ + -lco ROW_GROUP_SIZE=65536 +``` + +!!! note + + GDAL can also sort the features with `-lco SORT_BY_BBOX=YES`, but in our tests the Hilbert curve sort gave a better spatial grouping: for a tile covering ~1% of a dataset of 1M random points (16 row groups), 1 row group was read with the Hilbert sort and 5 with `SORT_BY_BBOX`. + +With [GeoPandas](https://geopandas.org/en/stable/docs/reference/api/geopandas.GeoDataFrame.to_parquet.html): + +```python +import geopandas + +gdf = geopandas.read_parquet("parcels.parquet") +gdf = gdf.iloc[gdf.hilbert_distance().argsort()] +gdf.to_parquet("parcels_sorted.parquet", write_covering_bbox=True, row_group_size=65536) +``` + +!!! warning + + DuckDB's `COPY ... TO` doesn't write the bbox `covering` metadata, and adding it with `KV_METADATA` results in two `geo` metadata entries in the file. Use DuckDB to sort the data, but GDAL or GeoPandas to write the final GeoParquet file with a bbox covering. + + DuckDB can write the native Parquet `GEOMETRY` type, with bounding box statistics (`GEOPARQUET_VERSION 'V2'`), but those statistics are not used when the file is read through a view, which is how `TiPg` registers parquet `sources`. + +#### Low zoom levels + +A good layout only helps when a tile covers a small part of the dataset: a tile at zoom level 5 over the whole dataset still reads every row group, and only returns the first `TIPG_MAX_FEATURES_PER_TILE` features. For large datasets, only request tiles at higher zoom levels (e.g with the `minZoom` option of your map library's source). diff --git a/docs/src/api/tipg/duck/collections.md b/docs/src/api/tipg/duck/collections.md new file mode 100644 index 00000000..13d62d68 --- /dev/null +++ b/docs/src/api/tipg/duck/collections.md @@ -0,0 +1 @@ +::: tipg.duck.collections diff --git a/docs/src/api/tipg/duck/database.md b/docs/src/api/tipg/duck/database.md new file mode 100644 index 00000000..7508272e --- /dev/null +++ b/docs/src/api/tipg/duck/database.md @@ -0,0 +1 @@ +::: tipg.duck.database diff --git a/docs/src/api/tipg/duck/settings.md b/docs/src/api/tipg/duck/settings.md new file mode 100644 index 00000000..fdd22381 --- /dev/null +++ b/docs/src/api/tipg/duck/settings.md @@ -0,0 +1 @@ +::: tipg.duck.settings diff --git a/examples/duckdb/.env-example b/examples/duckdb/.env-example new file mode 100644 index 00000000..a8509b7a --- /dev/null +++ b/examples/duckdb/.env-example @@ -0,0 +1,3 @@ +# TIPG_DUCKDB_DATABASE="source_coop.duckdb" +TIPG_DUCKDB_SOURCES='{"buildings": "https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' +TIPG_DEFAULT_MINZOOM=11 \ No newline at end of file diff --git a/examples/duckdb/build_source_coop_db.py b/examples/duckdb/build_source_coop_db.py new file mode 100644 index 00000000..a6484cd2 --- /dev/null +++ b/examples/duckdb/build_source_coop_db.py @@ -0,0 +1,201 @@ +"""Create a DuckDB database with one view per Source Cooperative GeoParquet file. + +The sources are listed in `source_coop_geoparquet.md` (markdown table with the +product name, title and an example file of each Source Cooperative product). + +Views read the remote files directly (no data is copied), using the same form +as tipg file `sources`: + + CREATE VIEW main. AS + SELECT * FROM read_parquet('https://data.source.coop//', file_row_number = true) + +So the tipg DuckDB catalog reads their GeoParquet metadata (spatial extent, +geometry type, bbox covering) and uses `file_row_number` as feature id. + +The remote file footers are read in parallel (`--threads`), then the views are +created sequentially from DuckDB's parquet metadata cache. + +Usage: + + python build_source_coop_db.py [source_coop_geoparquet.md] [source_coop.duckdb] [--threads 16] + + TIPG_DUCKDB_DATABASE=source_coop.duckdb uvicorn main:app + +""" + +import argparse +import pathlib +import re +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import duckdb + +BASE_URL = "https://data.source.coop" +HERE = pathlib.Path(__file__).parent + + +def quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def quote_ident(value: str) -> str: + """Quote a SQL identifier.""" + return '"' + value.replace('"', '""') + '"' + + +def parse_table(path: pathlib.Path) -> list[dict[str, str]]: + """Parse the rows of the markdown table. + + Columns: # | Product | Title | Source Cooperative | Example file | ... + """ + rows = [] + for line in path.read_text().splitlines(): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if len(cells) < 5 or not cells[0].isdigit(): + continue + + rows.append( + { + "product": cells[1].strip("`"), + "title": cells[2], + "file": cells[4].strip("`"), + } + ) + + return rows + + +def view_name(product: str) -> str: + """View name from the product name (e.g `fiboa/es-an` -> `fiboa_es_an`).""" + return re.sub(r"[^0-9a-zA-Z]+", "_", product).strip("_").lower() + + +def create_view( + conn: duckdb.DuckDBPyConnection, name: str, url: str, title: str +) -> list[str]: + """Create a view reading a remote parquet file, return its column types. + + Binding the view reads the remote file (HTTP requests), which is the + slow part. Thread safe: uses its own cursor (connection). + """ + cursor = conn.cursor() + try: + view = f"CREATE OR REPLACE VIEW main.{quote_ident(name)} AS" + try: + cursor.execute( + f"{view} SELECT * FROM read_parquet({quote_literal(url)}, file_row_number = true)" + ) + except duckdb.BinderException: + # the file already has a `file_row_number` column + cursor.execute(f"{view} SELECT * FROM read_parquet({quote_literal(url)})") + + cursor.execute( + f"COMMENT ON VIEW main.{quote_ident(name)} IS {quote_literal(title)}" + ) + return [ + r[1] + for r in cursor.execute(f"DESCRIBE main.{quote_ident(name)}").fetchall() + ] + finally: + cursor.close() + + +def main(md_path: pathlib.Path, db_path: pathlib.Path, threads: int) -> None: + """Create the database.""" + rows = parse_table(md_path) + print(f"{len(rows)} sources") + + conn = duckdb.connect(str(db_path)) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + conn.execute("SET parquet_metadata_cache = true") + # cache HTTP HEAD responses (file size, last modified) too + conn.execute("SET enable_http_metadata_cache = true") + + sources = { + view_name(row["product"]): (row, f"{BASE_URL}/{row['product']}/{row['file']}") + for row in rows + } + + created, failed, nongeo, conflicts = [], [], [], [] + + def _done(name: str, types: list[str]) -> None: + if not any(t.upper().startswith("GEOMETRY") for t in types): + nongeo.append(name) + created.append(name) + print(f"ok {name}", flush=True) + + def _failed(name: str, url: str, e: Exception) -> None: + error = str(e).splitlines()[0] + failed.append((name, url, error)) + print(f"FAILED {name}: {error}", flush=True) + + # Create the views in parallel (each one in its own transaction) + start = time.time() + with ThreadPoolExecutor(max_workers=threads) as pool: + futures = { + pool.submit(create_view, conn, name, url, row["title"]): name + for name, (row, url) in sources.items() + } + for future in as_completed(futures): + name = futures[future] + try: + _done(name, future.result()) + except duckdb.TransactionException: + # concurrent catalog writes conflicted: retry sequentially + conflicts.append(name) + except duckdb.Error as e: + _failed(name, sources[name][1], e) + + for name in conflicts: + row, url = sources[name] + try: + _done(name, create_view(conn, name, url, row["title"])) + except duckdb.Error as e: + _failed(name, url, e) + + conn.close() + + print(f"\ndone in {time.time() - start:.0f}s ({len(conflicts)} retried)") + print_report(created, failed, nongeo) + + +def print_report( + created: list[str], failed: list[tuple[str, str, str]], nongeo: list[str] +) -> None: + """Print the summary.""" + print(f"\n{len(created)} views created, {len(failed)} failed") + if nongeo: + print( + f"no GEOMETRY column (hidden by `only_spatial_tables`): {', '.join(nongeo)}" + ) + for name, url, error in failed: + print(f" {name}: {url}\n {error}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "sources", + nargs="?", + type=pathlib.Path, + default=HERE / "source_coop_geoparquet.md", + help="markdown table of the sources", + ) + parser.add_argument( + "database", + nargs="?", + type=pathlib.Path, + default=HERE / "source_coop.duckdb", + help="output DuckDB database", + ) + parser.add_argument( + "--threads", + type=int, + default=16, + help="number of files read concurrently (default: 16)", + ) + args = parser.parse_args() + main(args.sources, args.database, args.threads) diff --git a/examples/duckdb/main.py b/examples/duckdb/main.py new file mode 100644 index 00000000..476f55d9 --- /dev/null +++ b/examples/duckdb/main.py @@ -0,0 +1,118 @@ +"""tipg app with DuckDB backend. + +export TIPG_DUCKDB_SOURCES='{"test":"https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' +uv run --group server uvicorn main:app + +""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from starlette.middleware.cors import CORSMiddleware +from starlette_cramjam.middleware import CompressionMiddleware + +from tipg import __version__ as tipg_version +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.extensions.viewer import viewerExtension +from tipg.factory import Endpoints +from tipg.middleware import CacheControlMiddleware +from tipg.openapi import _update_openapi +from tipg.settings import APISettings + +settings = APISettings() + +duckdb_catalog_settings = DuckDBCatalogSettings() + + +async def register_collection_catalog( + app: FastAPI, + catalog_settings: DuckDBCatalogSettings, +) -> None: + """Register Table catalog.""" + duck_collections = await duck_get_collection_index( + connection=app.state.duckdb_connection, + settings=catalog_settings, + ) + + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + # Create Connection Pool + await connect_to_duckdb(app) + + # Register Collection Catalog + await register_collection_catalog( + app, + catalog_settings=duckdb_catalog_settings, + ) + + yield + + # Close the Connection Pool + await close_duckdb_connection(app) + + +app = FastAPI( + title=settings.name, + version=tipg_version, + openapi_url="/api", + docs_url="/api.html", + lifespan=lifespan, + root_path=settings.root_path, +) + +# Fix OpenAPI response header for OGC Common compatibility +_update_openapi(app) + +ogc_api = Endpoints( + title=settings.name, + with_tiles_viewer=False, + extensions=[viewerExtension()], +) +app.include_router(ogc_api.router) + + +# Set all CORS enabled origins +if settings.cors_origins: + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["GET"], + allow_headers=["*"], + ) + +app.add_middleware( + CacheControlMiddleware, + cachecontrol=settings.cachecontrol, + exclude_path={r"^/healthz$", r"^/metrics$"}, +) +app.add_middleware(CompressionMiddleware, compression_level=6) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) + + +@app.get( + "/healthz", + description="Health Check.", + summary="Health Check.", + operation_id="healthCheck", + tags=["Health Check"], +) +def ping(): + """Health check.""" + return {"ping": "pong!"} diff --git a/examples/duckdb/main_external.py b/examples/duckdb/main_external.py new file mode 100644 index 00000000..ffa98c92 --- /dev/null +++ b/examples/duckdb/main_external.py @@ -0,0 +1,169 @@ +"""tipg app with DuckDB backend. + +uv run --group server uvicorn main_external:app + +http://127.0.0.1:8000/collections/external/items?url=https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet + +""" + +import datetime +import hashlib +from contextlib import asynccontextmanager +from typing import Annotated, cast + +import duckdb +from fastapi import FastAPI, HTTPException, Path, Query, Request +from starlette.concurrency import run_in_threadpool +from starlette.middleware.cors import CORSMiddleware +from starlette_cramjam.middleware import CompressionMiddleware + +from tipg import __version__ as tipg_version +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + DuckDBSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) +from tipg.duck.collections import DuckCollection +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.extensions.viewer import viewerExtension +from tipg.factory import Endpoints +from tipg.middleware import CacheControlMiddleware +from tipg.openapi import _update_openapi +from tipg.settings import APISettings + +settings = APISettings() + +duckdb_catalog_settings = DuckDBCatalogSettings() + + +ALLOWED_SCHEMES = ("http://", "https://", "s3://") + + +def _quote_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +async def CollectionParams( + request: Request, + collectionId: Annotated[str, Path(description="Collection identifier")], + url: Annotated[str, Query(description="URL of the Geoparquet dataset")], +) -> DuckCollection: + """Return Layer Object.""" + if collectionId != "external": + raise HTTPException(404, f"Collection '{collectionId}' not found.") + + # Don't let users read local files (`/etc/...`, `file://`) or arbitrary formats + if not url.startswith(ALLOWED_SCHEMES) or not url.lower().endswith( + (".parquet", ".geoparquet") + ): + raise HTTPException(422, "`url` must be a remote (geo)parquet file.") + + cache: dict[str, DuckCollection] = request.app.state.external_collections + if collection := cache.get(url): + return collection + + # dynamically create DuckCollection based on the URL: + # 1. register the file as a view in the in-memory database + # 2. let tipg's catalog builder introspect that single view + name = "ext_" + hashlib.sha256(url.encode()).hexdigest()[:16] + conn = request.app.state.duckdb_connection + view = f'CREATE OR REPLACE VIEW memory.main."{name}" AS SELECT * FROM read_parquet({_quote_literal(url)}' + try: + try: + # file row number as stable feature id (same as tipg's `sources`) + await run_in_threadpool(conn.execute, f"{view}, file_row_number = true)") + except duckdb.BinderException: + await run_in_threadpool(conn.execute, f"{view})") + except duckdb.Error as e: + raise HTTPException(422, f"Could not read {url}: {e}") from e + + collections = await duck_get_collection_index( + connection=conn, + settings=duckdb_catalog_settings.model_copy( + update={"schemas": ["main"], "tables": [f"main.{name}"]} + ), + ) + if not collections: + raise HTTPException(422, f"No spatial collection found in {url}.") + + # expose it as `external` so links point back to `/collections/external/...` + collection = cast( + DuckCollection, collections[0].model_copy(update={"id": collectionId}) + ) + cache[url] = collection + return collection + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + # Create Connection Pool + await connect_to_duckdb(app, settings=DuckDBSettings(database=None, sources={})) + + # cache of url -> DuckCollection + app.state.external_collections = {} + # empty catalog so `/collections` doesn't error + app.state.collection_catalog = Catalog( + collections={}, last_updated=datetime.datetime.now() + ) + + yield + + # Close the Connection Pool + await close_duckdb_connection(app) + + +app = FastAPI( + title=settings.name, + version=tipg_version, + openapi_url="/api", + docs_url="/api.html", + lifespan=lifespan, + root_path=settings.root_path, +) + +# Fix OpenAPI response header for OGC Common compatibility +_update_openapi(app) + +ogc_api = Endpoints( + title=settings.name, + with_tiles_viewer=False, + extensions=[viewerExtension()], + collection_dependency=CollectionParams, # type: ignore +) +app.include_router(ogc_api.router) + + +# Set all CORS enabled origins +if settings.cors_origins: + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["GET"], + allow_headers=["*"], + ) + +app.add_middleware( + CacheControlMiddleware, + cachecontrol=settings.cachecontrol, + exclude_path={r"^/healthz$", r"^/metrics$"}, +) +app.add_middleware(CompressionMiddleware, compression_level=6) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) + + +@app.get( + "/healthz", + description="Health Check.", + summary="Health Check.", + operation_id="healthCheck", + tags=["Health Check"], +) +def ping(): + """Health check.""" + return {"ping": "pong!"} diff --git a/examples/duckdb/source_coop.duckdb b/examples/duckdb/source_coop.duckdb new file mode 100644 index 00000000..1e465511 Binary files /dev/null and b/examples/duckdb/source_coop.duckdb differ diff --git a/examples/duckdb/source_coop_geoparquet.md b/examples/duckdb/source_coop_geoparquet.md new file mode 100644 index 00000000..a4124a87 --- /dev/null +++ b/examples/duckdb/source_coop_geoparquet.md @@ -0,0 +1,123 @@ +| # | Product | Title | Source Cooperative | Example file | GeoParquet | Geometry column | Rows (example file) | +|---|---|---|---|---|---|---|---| +| 1 | `addresscloud/epc` | Energy Performance Certificate Ratings (Domestic) - England and Wales | https://source.coop/addresscloud/epc | `geoparquet-local-authority/Adur.parquet` | 1.0.0 | `geometry` | 24,228 | +| 2 | `ausantarctic/ghrsst-mur-v2` | GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis (v4.1) | https://source.coop/ausantarctic/ghrsst-mur-v2 | `ghrsst-mur-v2.parquet` | 1.1.0 | `geometry` | 8,927 | +| 3 | `cboettig/conservation-policy` | Conservation Policy Spending in Federal Legislation | https://source.coop/cboettig/conservation-policy | `Inflation_Reduction_Act_Projects.parquet` | 1.0.0 | `geometry` | 813 | +| 4 | `cboettig/fire` | Wildfire Boundary Polygons from USGS and CalFire in cloud native formats | https://source.coop/cboettig/fire | `calfire-2024/firep.parquet` | 1.0.0 | `shape` | 22,810 | +| 5 | `cboettig/habitat-corridors` | Wildlife Habitat Corridors | https://source.coop/cboettig/habitat-corridors | `ungulate_migration.parquet` | 1.0.0 | `geometry` | 64 | +| 6 | `cboettig/justice40` | Climate and Economic Justice Screening Tool (cloud-native formats) | https://source.coop/cboettig/justice40 | `disadvantaged-communities.parquet` | 1.0.0 | `SHAPE` | 74,045 | +| 7 | `cboettig/social-vulnerability` | Social Vulnerability Index | https://source.coop/cboettig/social-vulnerability | `2000/SVI2000_US_county.parquet` | 1.0.0 | `Shape` | 3,141 | +| 8 | `cboettig/us-boundaries` | US Political Boundaries, Including Domestic Sovereign Nations (Federally-Recognized Tribes) | https://source.coop/cboettig/us-boundaries | `mappinginequality.parquet` | 1.1.0 | `geom` | 10,154 | +| 9 | `cecil/forest-carbon-boundaries` | Forest Carbon Project Boundaries | https://source.coop/cecil/forest-carbon-boundaries | `africa.parquet` | 1.1.0 | `geometry` | 72 | +| 10 | `cholmes/admin-boundaries` | Open Administrative Boundaries | https://source.coop/cholmes/admin-boundaries | `countries.parquet` | 1.1.0 | `geometry` | 219 | +| 11 | `cholmes/eurocrops` | EuroCrops | https://source.coop/cholmes/eurocrops | `eurocrops-harmonized-only.parquet` | 1.0.0 | `geometry` | 22,783,273 | +| 12 | `cholmes/google-open-buildings` | Google Open Buildings | https://source.coop/cholmes/google-open-buildings | `geoparquet-by-country/country_iso=AF/AF.parquet` | 1.0.0-beta.1 | `geometry` | 3,691 | +| 13 | `cholmes/nyc-taxi-zones` | Taxi Zones from TLC Trip Record Data | https://source.coop/cholmes/nyc-taxi-zones | `taxi_zones.parquet` | 1.0.0-beta.1 | `geometry` | 263 | +| 14 | `cholmes/overture` | Overture Open Buildings | https://source.coop/cholmes/overture | `geoparquet-country-quad-2/AD.parquet` | 1.0.0-beta.1 | `geometry` | 5,747 | +| 15 | `clay/clay-v1-5-naip-2` | NAIP data embedded with Clay v1.5 (rev2) | https://source.coop/clay/clay-v1-5-naip-2 | `al/2011/100cm/rgbir_cog/30085/m_3008501_ne_16_1_20110815.parquet` | 1.0.0 | `geometry` | 725 | +| 16 | `clay/clay-v1-5-sentinel2` | Sentinel-2 data embedded with Clay v1.5 | https://source.coop/clay/clay-v1-5-sentinel2 | `sentinel-s2-l2a-cogs/1/U/CS/2024/10/S2A_1UCS_20241005_0_L2A/S2A_1UCS_20241005_0_L2A.parquet` | 1.0.0 | `geometry` | 874 | +| 17 | `earthgenome/earthindexembeddings` | Earth Index Embeddings | https://source.coop/earthgenome/earthindexembeddings | `2024/01GDM_2024-01-01_2025-01-01.parquet` | 1.0.0 | `geometry` | 234,340 | +| 18 | `earthgenome/food-twin` | Global Food Twin | https://source.coop/earthgenome/food-twin | `inputs/infrastructure/nodes.geoparquet` | 0.4.0 | `geometry` | 21,553,314 | +| 19 | `fiboa/ai4sf` | Field boundaries for Cambodia and Vietnam (AI4SmallFarms) | https://source.coop/fiboa/ai4sf | `ai4sf.parquet` | 1.0.0 | `geometry` | 439,001 | +| 20 | `fiboa/at-crop` | Crop Fields for Austria | https://source.coop/fiboa/at-crop | `at_crop_2020.parquet` | 1.1.0 | `geometry` | 2,614,636 | +| 21 | `fiboa/austria` | Field boundaries for Austria | https://source.coop/fiboa/austria | `inspire_referenzen_2021.parquet` | 1.0.0 | `geometry` | 1,299,745 | +| 22 | `fiboa/be-vlg` | Crop field boundaries for Belgium - Flanders | https://source.coop/fiboa/be-vlg | `be_vlg.parquet` | 1.0.0 | `geometry` | 588,193 | +| 23 | `fiboa/be-wa` | Field boundaries for Walonia, Belgium | https://source.coop/fiboa/be-wa | `be_wa.parquet` | 1.1.0 | `geometry` | 341,968 | +| 24 | `fiboa/br-ba-lem` | Field Boundaries for Luís Eduardo Magalhães in the west of Bahia state, Brazil | https://source.coop/fiboa/br-ba-lem | `br-ba-lem.parquet` | 1.0.0 | `geometry` | 1,854 | +| 25 | `fiboa/croatia` | Croatian Agricultural Spatial Data | https://source.coop/fiboa/croatia | `croatia.parquet` | 1.1.0 | `geometry` | 1,346,655 | +| 26 | `fiboa/czech` | Czech crop fields | https://source.coop/fiboa/czech | `czech.parquet` | 1.1.0 | `geometry` | 426,584 | +| 27 | `fiboa/data` | Fiboa datasets | https://source.coop/fiboa/data | `at/at-2025.parquet` | 1.1.0 | `geometry` | 2,944,405 | +| 28 | `fiboa/de-bb` | Field boundaries for Berlin / Brandenburg, Germany | https://source.coop/fiboa/de-bb | `dfbk.parquet` | 1.0.0 | `geometry` | 90,770 | +| 29 | `fiboa/de-mv` | Field boundaries for Mecklenburg-Western Pomerania, Germany | https://source.coop/fiboa/de-mv | `de-mv.parquet` | 1.1.0 | `geometry` | 70,941 | +| 30 | `fiboa/de-nds` | Field boundaries for Lower Saxony / Hamburg / Bremen, Germany | https://source.coop/fiboa/de-nds | `FB_NDS.parquet` | 1.0.0 | `geometry` | 547,196 | +| 31 | `fiboa/de-nrw` | Field Boundaries for North Rhine-Westphalia (NRW), Germany | https://source.coop/fiboa/de-nrw | `LFK-AKTI_EPSG25832.parquet` | 1.0.0 | `geometry` | 399,725 | +| 32 | `fiboa/de-sax` | Field boundaries for Saxony, Germany | https://source.coop/fiboa/de-sax | `de-sax.parquet` | 1.1.0 | `geometry` | 95,510 | +| 33 | `fiboa/de-sh` | Field Boundaries for Schleswig-Holstein (SH), Germany | https://source.coop/fiboa/de-sh | `Feldbloecke_2024.parquet` | 1.0.0 | `geometry` | 197,673 | +| 34 | `fiboa/de-sl` | Field boundaries for Saarland, Germany | https://source.coop/fiboa/de-sl | `de-sl.parquet` | 1.1.0 | `geometry` | 53,366 | +| 35 | `fiboa/de-th` | Field boundaries for Thuringia, Germany | https://source.coop/fiboa/de-th | `de-th.parquet` | 1.0.0 | `geometry` | 101,216 | +| 36 | `fiboa/denmark` | Denmark Crop Fields (Marker) | https://source.coop/fiboa/denmark | `denmark.parquet` | 1.0.0 | `geometry` | 610,180 | +| 37 | `fiboa/ec-lv` | Field Boundaries for Latvia, provided by Eurocrops | https://source.coop/fiboa/ec-lv | `ec_lv.parquet` | 1.0.0 | `geometry` | 432,188 | +| 38 | `fiboa/es-an` | Spain Andalusia Crop fields | https://source.coop/fiboa/es-an | `es_an.parquet` | 1.1.0 | `geometry` | 4,381,464 | +| 39 | `fiboa/es-ar` | SIGPAC - Sistema de Información Geográfica de la Política Agrícola común (ejercicio actual) | https://source.coop/fiboa/es-ar | `es_ar.parquet` | 1.1.0 | `geometry` | 4,146,895 | +| 40 | `fiboa/es-cat` | Catalonia Crop Fields (Mapa de cultius) | https://source.coop/fiboa/es-cat | `es_cat.parquet` | 1.1.0 | `geometry` | 733,072 | +| 41 | `fiboa/es-cb` | Spain Cantabria Crop fields | https://source.coop/fiboa/es-cb | `es_cb.parquet` | 1.1.0 | `geometry` | 605,149 | +| 42 | `fiboa/es-cl` | Spain Castile and León Crop fields (SIGPAC) | https://source.coop/fiboa/es-cl | `es_cl.parquet` | 1.1.0 | `geometry` | 10,232,880 | +| 43 | `fiboa/es-cm` | Spain Castilla-La Mancha Crop fields | https://source.coop/fiboa/es-cm | `es_cm.parquet` | 1.1.0 | `geometry` | 6,453,236 | +| 44 | `fiboa/es-cn` | Spain Canary Islands Crop fields | https://source.coop/fiboa/es-cn | `es_cn.parquet` | 1.1.0 | `geometry` | 466,169 | +| 45 | `fiboa/es-ex` | Spain Extremadura Crop fields | https://source.coop/fiboa/es-ex | `es_ex.parquet` | 1.1.0 | `geometry` | 2,070,876 | +| 46 | `fiboa/es-ga` | Gallicia SIXPAC Crop Fields | https://source.coop/fiboa/es-ga | `es_ga.parquet` | 1.1.0 | `geometry` | 4,963,127 | +| 47 | `fiboa/es-md` | Comunidad de Madrid Crop Fields | https://source.coop/fiboa/es-md | `es_md.parquet` | 1.1.0 | `geometry` | 631,861 | +| 48 | `fiboa/es-nc` | Spain Navarra Crop fields | https://source.coop/fiboa/es-nc | `es_nc.parquet` | 1.1.0 | `geometry` | 663,556 | +| 49 | `fiboa/es-pm` | Spain Balearic Islands Crop fields | https://source.coop/fiboa/es-pm | `es_pm.parquet` | 1.1.0 | `geometry` | 394,623 | +| 50 | `fiboa/es-pv` | Spain Basque Country Crop Fields | https://source.coop/fiboa/es-pv | `es_pv.parquet` | 1.1.0 | `geometry` | 562,407 | +| 51 | `fiboa/es-vc` | Spain Agricultural Plots in the Valencian Community (SIGPAC) | https://source.coop/fiboa/es-vc | `es_vc.parquet` | 1.1.0 | `geometry` | 2,319,893 | +| 52 | `fiboa/estonia-ec` | Field Boundaries for Estonia, provided by Eurocrops | https://source.coop/fiboa/estonia-ec | `data/ec_ee.parquet` | 1.1.0 | `geometry` | 176,064 | +| 53 | `fiboa/finland` | Finnish Crop Fields (Maatalousmaa) | https://source.coop/fiboa/finland | `finland.parquet` | 1.0.0 | `geometry` | 1,086,825 | +| 54 | `fiboa/france-ec` | Field Boundaries for France, provided by Eurocrops | https://source.coop/fiboa/france-ec | `france_eurocrops_2018_fiboa.parquet` | 1.0.0 | `geometry` | 9,517,878 | +| 55 | `fiboa/ireland` | Ireland Crop Fields | https://source.coop/fiboa/ireland | `ireland.parquet` | 1.1.0 | `geometry` | 1,027,439 | +| 56 | `fiboa/japan` | Japan Fude Polygons | https://source.coop/fiboa/japan | `japan.parquet` | 1.1.0 | `geometry` | 29,418,076 | +| 57 | `fiboa/lacunalabels` | Lacuna Labels | https://source.coop/fiboa/lacunalabels | `lacunalabels.parquet` | 1.1.0 | `geometry` | 825,395 | +| 58 | `fiboa/latvia` | Field Boundaries for Latvia | https://source.coop/fiboa/latvia | `latvia.parquet` | 1.1.0 | `geometry` | 420,863 | +| 59 | `fiboa/luxembourg` | Luxembourg FLIK Parcels | https://source.coop/fiboa/luxembourg | `luxembourg.parquet` | 1.1.0 | `geometry` | 87,997 | +| 60 | `fiboa/newzealand` | New Zealand National irrigated land | https://source.coop/fiboa/newzealand | `newzealand.parquet` | 1.1.0 | `geometry` | 42,133 | +| 61 | `fiboa/nl-crop` | Crop fields for The Netherlands | https://source.coop/fiboa/nl-crop | `nl_crop.parquet` | 1.0.0 | `geometry` | 1,350,739 | +| 62 | `fiboa/nl-ref` | Reference Field boundaries, The Netherlands | https://source.coop/fiboa/nl-ref | `nl_ref.parquet` | 1.0.0 | `geometry` | 549,218 | +| 63 | `fiboa/portugal` | Field boundaries for Portugal | https://source.coop/fiboa/portugal | `Continente.parquet` | 1.0.0 | `geometry` | 4,804,494 | +| 64 | `fiboa/slovakia` | Slovakia Crop fields (Hranice užívania) | https://source.coop/fiboa/slovakia | `slovakia.parquet` | 1.1.0 | `geometry` | 274,467 | +| 65 | `fiboa/slovenia` | Crop fields for Slovenia | https://source.coop/fiboa/slovenia | `slovenia.parquet` | 1.1.0 | `geometry` | 830,856 | +| 66 | `fiboa/slovenia-ec` | Field Boundaries for Slovenia, provided by EuroCrops | https://source.coop/fiboa/slovenia-ec | `data/slovenia_eurocrops_2021_fiboa.parquet` | 1.0.0 | `geometry` | 828,161 | +| 67 | `fiboa/sweden` | Field boundaries for Sweden | https://source.coop/fiboa/sweden | `sweden.parquet` | 1.1.0 | `geometry` | 1,142,785 | +| 68 | `fiboa/switzerland` | Field boundaries for Switzerland | https://source.coop/fiboa/switzerland | `switzerland.parquet` | 1.1.0 | `geometry` | 1,159,957 | +| 69 | `fiboa/us-ca-scm` | California Statewide Crop Mapping | https://source.coop/fiboa/us-ca-scm | `us_ca_scm.parquet` | 1.1.0 | `geometry` | 438,790 | +| 70 | `fiboa/us-usda-cropland` | USDA Crop Sequence Boundaries | https://source.coop/fiboa/us-usda-cropland | `us_usda_cropland.parquet` | 1.1.0 | `geometry` | 16,194,865 | +| 71 | `fika/waternet` | WaterNet Global Waterways | https://source.coop/fika/waternet | `vector/1020000010.parquet` | 1.0.0 | `geometry` | 4,455,083 | +| 72 | `fused/fsq-os-places` | Foursquare Open Source Places - Fused-partitioned | https://source.coop/fused/fsq-os-places | `2024-11-19/places/0.parquet` | 1.0.0-beta.1 | `geometry` | 111,406 | +| 73 | `fused/overture` | Overture Maps - Fused-partitioned | https://source.coop/fused/overture | `2024-02-15-alpha-0/theme=admins/type=administrativeBoundary/0.parquet` | 1.0.0-beta.1 | `geometry` | 2,587 | +| 74 | `giswqs/depressions` | National Surface Depressions | https://source.coop/giswqs/depressions | `all_dep/HU2_01.parquet` | 1.0.0-beta.1 | `geometry` | 726,727 | +| 75 | `giswqs/giw` | Non-Floodplain Wetlands | https://source.coop/giswqs/giw | `wetlands/AL_IW.parquet` | 1.0.0-beta.1 | `geometry` | 87,653 | +| 76 | `giswqs/nwi` | National Wetlands Inventory | https://source.coop/giswqs/nwi | `historic_metadata/CA_Historic_Wetlands_Project_Metadata.parquet` | 1.0.0-beta.1 | `geometry` | 0 | +| 77 | `kerner-lab/fields-of-the-world` | Fields of The World (FTW) | https://source.coop/kerner-lab/fields-of-the-world | `austria/chips_austria.parquet` | 1.0.0 | `geometry` | 6,686 | +| 78 | `kerner-lab/fields-of-the-world-austria` | Fields of The World: Field Boundaries for Austria | https://source.coop/kerner-lab/fields-of-the-world-austria | `boundaries_austria_2021.parquet` | 1.0.0 | `geometry` | 196,101 | +| 79 | `kerner-lab/fields-of-the-world-belgium` | Fields of The World: Field Boundaries for Belgium | https://source.coop/kerner-lab/fields-of-the-world-belgium | `boundaries_belgium_2021.parquet` | 1.0.0 | `geometry` | 63,431 | +| 80 | `kerner-lab/fields-of-the-world-brazil` | Fields of The World: Field Boundaries for Brazil | https://source.coop/kerner-lab/fields-of-the-world-brazil | `boundaries_brazil_2020.parquet` | 1.0.0 | `geometry` | 1,854 | +| 81 | `kerner-lab/fields-of-the-world-cambodia` | Fields of The World: Field Boundaries for Cambodia | https://source.coop/kerner-lab/fields-of-the-world-cambodia | `boundaries_cambodia_2021.parquet` | 1.0.0 | `geometry` | 318,088 | +| 82 | `kerner-lab/fields-of-the-world-corsica` | Fields of The World: Field Boundaries for Corsica | https://source.coop/kerner-lab/fields-of-the-world-corsica | `boundaries_corsica_2021.parquet` | 1.0.0 | `geometry` | 5,360 | +| 83 | `kerner-lab/fields-of-the-world-croatia` | Fields of The World: Field Boundaries for Croatia | https://source.coop/kerner-lab/fields-of-the-world-croatia | `boundaries_croatia_2023.parquet` | 1.0.0 | `geometry` | 157,481 | +| 84 | `kerner-lab/fields-of-the-world-denmark` | Fields of The World: Field Boundaries for Denmark | https://source.coop/kerner-lab/fields-of-the-world-denmark | `boundaries_denmark_2021.parquet` | 1.0.0 | `geometry` | 37,677 | +| 85 | `kerner-lab/fields-of-the-world-estonia` | Fields of The World: Field Boundaries for Estonia | https://source.coop/kerner-lab/fields-of-the-world-estonia | `boundaries_estonia_2021.parquet` | 1.0.0 | `geometry` | 26,695 | +| 86 | `kerner-lab/fields-of-the-world-finland` | Fields of The World: Field Boundaries for Finland | https://source.coop/kerner-lab/fields-of-the-world-finland | `boundaries_finland_2021.parquet` | 1.0.0 | `geometry` | 57,323 | +| 87 | `kerner-lab/fields-of-the-world-france` | Fields of The World: Field Boundaries for France | https://source.coop/kerner-lab/fields-of-the-world-france | `boundaries_france_2020.parquet` | 1.0.0 | `geometry` | 55,342 | +| 88 | `kerner-lab/fields-of-the-world-germany` | Fields of The World: Field Boundaries for Germany | https://source.coop/kerner-lab/fields-of-the-world-germany | `boundaries_germany_2018_2019.parquet` | 1.0.0 | `geometry` | 4,598 | +| 89 | `kerner-lab/fields-of-the-world-india` | Fields of The World: Field Boundaries for India | https://source.coop/kerner-lab/fields-of-the-world-india | `boundaries_india_2016.parquet` | 1.0.0 | `geometry` | 10,013 | +| 90 | `kerner-lab/fields-of-the-world-kenya` | Fields of The World: Field Boundaries for Kenya | https://source.coop/kerner-lab/fields-of-the-world-kenya | `boundaries_kenya_2022.parquet` | 1.0.0 | `geometry` | 874 | +| 91 | `kerner-lab/fields-of-the-world-latvia` | Fields of The World: Field Boundaries for Latvia | https://source.coop/kerner-lab/fields-of-the-world-latvia | `boundaries_latvia_2021.parquet` | 1.0.0 | `geometry` | 44,964 | +| 92 | `kerner-lab/fields-of-the-world-lithuania` | Fields of The World: Field Boundaries for Lithuania | https://source.coop/kerner-lab/fields-of-the-world-lithuania | `boundaries_lithuania_2021.parquet` | 1.0.0 | `geometry` | 61,424 | +| 93 | `kerner-lab/fields-of-the-world-luxembourg` | Fields of The World: Field Boundaries for Luxembourg | https://source.coop/kerner-lab/fields-of-the-world-luxembourg | `boundaries_luxembourg_2022.parquet` | 1.0.0 | `geometry` | 29,018 | +| 94 | `kerner-lab/fields-of-the-world-netherlands` | Fields of The World: Field Boundaries for Netherlands | https://source.coop/kerner-lab/fields-of-the-world-netherlands | `boundaries_netherlands_2022.parquet` | 1.0.0 | `geometry` | 43,169 | +| 95 | `kerner-lab/fields-of-the-world-portugal` | Fields of The World: Field Boundaries for Portugal | https://source.coop/kerner-lab/fields-of-the-world-portugal | `boundaries_portugal_2021.parquet` | 1.0.0 | `geometry` | 5,040 | +| 96 | `kerner-lab/fields-of-the-world-rwanda` | Fields of The World: Field Boundaries for Rwanda | https://source.coop/kerner-lab/fields-of-the-world-rwanda | `boundaries_rwanda_2021.parquet` | 1.0.0 | `geometry` | 1,532 | +| 97 | `kerner-lab/fields-of-the-world-slovakia` | Fields of The World: Field Boundaries for Slovakia | https://source.coop/kerner-lab/fields-of-the-world-slovakia | `boundaries_slovakia_2021.parquet` | 1.0.0 | `geometry` | 14,242 | +| 98 | `kerner-lab/fields-of-the-world-slovenia` | Fields of The World: Field Boundaries for Slovenia | https://source.coop/kerner-lab/fields-of-the-world-slovenia | `boundaries_slovenia_2021.parquet` | 1.0.0 | `geometry` | 67,488 | +| 99 | `kerner-lab/fields-of-the-world-southafrica` | Fieldscapes: Field Boundaries for South Africa | https://source.coop/kerner-lab/fields-of-the-world-southafrica | `boundaries_south_africa_2018.parquet` | 1.0.0 | `geometry` | 6,568 | +| 100 | `kerner-lab/fields-of-the-world-spain` | Fields of The World: Field Boundaries for Spain | https://source.coop/kerner-lab/fields-of-the-world-spain | `boundaries_spain_2020.parquet` | 1.0.0 | `geometry` | 258,465 | +| 101 | `kerner-lab/fields-of-the-world-sweden` | Fields of The World: Field Boundaries for Sweden | https://source.coop/kerner-lab/fields-of-the-world-sweden | `boundaries_sweden_2021.parquet` | 1.0.0 | `geometry` | 39,718 | +| 102 | `kerner-lab/fields-of-the-world-vietnam` | Fields of The World: Field Boundaries for Vietnam | https://source.coop/kerner-lab/fields-of-the-world-vietnam | `boundaries_vietnam_2021.parquet` | 1.0.0 | `geometry` | 120,913 | +| 103 | `kerner-lab/fieldscapes-denmark` | Fieldscapes: Field Boundaries for Denmark | https://source.coop/kerner-lab/fieldscapes-denmark | `boundaries_denmark_2021.parquet` | 1.0.0 | `geometry` | 37,677 | +| 104 | `krishnaglodha/ksrsac-gis` | GIS Data by Karnataka Geographic Information System | https://source.coop/krishnaglodha/ksrsac-gis | `hobli/01_Belagavi.parquet` | 1.1.0 | `geometry` | 46 | +| 105 | `maxar/maxar-opendata` | Maxar Open Data Program (ARD) | https://source.coop/maxar/maxar-opendata | `maxar-opendata.parquet` | 1.0.0-beta.1 | `geometry` | 14,979 | +| 106 | `ordnancesurvey/ngd-boundaries` | OS National Geographic Database - Boundaries Collection in GeoParquet | https://source.coop/ordnancesurvey/ngd-boundaries | `GeoParquet/asu_bdy_boundaryhighwatermark.parquet` | 1.0.0-beta.1 | `geometry` | 13,208 | +| 107 | `pacificspatial/field-polygon-jp` | Japanese Farmland Parcel Polygons | https://source.coop/pacificspatial/field-polygon-jp | `parquet/jp_field_polygons_2021.parquet` | 1.0.0-dev | `GEOM` | 30,165,774 | +| 108 | `pacificspatial/flateau` | Flateau (Building footprint data in Japan) | https://source.coop/pacificspatial/flateau | `merged/merged_building_centroid_lod0.parquet` | 1.0.0-dev | `GEOM` | 18,513,605 | +| 109 | `pacificspatial/vegetation-jp` | Japanese Vegetation Map (1/25,000) | https://source.coop/pacificspatial/vegetation-jp | `parquet/jp_vegetation25000_en.parquet` | 1.0.0-dev | `GEOM` | 4,940,015 | +| 110 | `planet/eu-field-boundaries` | Agricultural Field Boundaries over EU | https://source.coop/planet/eu-field-boundaries | `by_country/AD.parquet` | 1.1.0 | `geometry` | 1,944 | +| 111 | `planet/rapidcrops` | RapidCrops: A pan-European label dataset for large-scale crop classification | https://source.coop/planet/rapidcrops | `combined/2018/country_iso=AUT/part-0.geo.parquet` | 1.1.0 | `geometry` | 2,315,115 | +| 112 | `sarahgamal/overture-places-riyadh` | Overture Maps - Places Dataset - Riyadh, Saudi Arabia | https://source.coop/sarahgamal/overture-places-riyadh | `riyadh_places.parquet` | 1.1.0 | `geometry` | 32,039 | +| 113 | `sarahgamal/riyadh-places` | Places - Riyadh City - Saudi Arabia | https://source.coop/sarahgamal/riyadh-places | `riyadh_places.parquet` | 1.0.0 | `geometry` | 8,836 | +| 114 | `streambatch/worldcereal` | ESA WorldCereal | https://source.coop/streambatch/worldcereal | `world_cereals_scaled_final_all.geoparquet` | 1.0.0-beta.1 | `geometry` | 6,827,917 | +| 115 | `tabaqat/riyadh-places` | Overture Maps - Places Dataset (GeoParquet) - Riyadh, Saudi Arabia | https://source.coop/tabaqat/riyadh-places | `riyadh_places.parquet` | 1.0.0 | `geometry` | 28,637 | +| 116 | `vida/google-microsoft-open-buildings` | Google-Microsoft Open Buildings - combined by VIDA | https://source.coop/vida/google-microsoft-open-buildings | `geoparquet/by_country/country_iso=AFG/AFG.parquet` | 1.1.0 | `geometry` | 7,840,655 | +| 117 | `vida/google-microsoft-osm-open-buildings` | Google-Microsoft-OSM Open Buildings - combined by VIDA | https://source.coop/vida/google-microsoft-osm-open-buildings | `geoparquet/by_country/country_iso=AFG/AFG.parquet` | 1.1.0 | `geometry` | 8,183,697 | +| 118 | `wherobots/usa-structures` | US Structures from Oak Ridge National Laboratory | https://source.coop/wherobots/usa-structures | `geoparquet/part-00000-170892fe-0fe0-43c1-999d-f0911ce43365-c000.zstd.parquet` | 1.1.0 | `geometry` | 12,965,123 | +| 119 | `youssef-harby/egms-copernicus` | European Ground Motion Cloud Native Data | https://source.coop/youssef-harby/egms-copernicus | `L2a/parquet/CY/EGMS_L2a_087_0205_IW1_VV_2019_2023_1.parquet` | 1.1.0 | `geom` | 440,200 | +| 120 | `youssef-harby/overture-maps-stac` | Overture Maps STAC | https://source.coop/youssef-harby/overture-maps-stac | `data_1.parquet` | 1.1.0-dev | `geometry` | 15,702 | +| 121 | `zluo43/citibike` | NYC Citi Bike | https://source.coop/zluo43/citibike | `new_schema_combined_with_geom.parquet/year=2019/month=1/data_0.parquet` | 1.1.0 | `start_geom` | 1 | diff --git a/plans/duckdb-backend.md b/plans/duckdb-backend.md new file mode 100644 index 00000000..f73224a3 --- /dev/null +++ b/plans/duckdb-backend.md @@ -0,0 +1,83 @@ +# DuckDB backend for tipg (`tipg.duck`) + +## Goal +Serve DuckDB tables/views and files (GeoParquet, `ST_Read`-able files) as OGC Features + Tiles collections, and allow mixing them with PostgreSQL collections in a single catalog. + +Scope: DuckDB database file + file sources; features + MVT tiles; no app module (`tipg/main.py` untouched). + +## DuckDB constraints the design relies on (DuckDB 2.0 + spatial) +- cql2 `to_sql()` output runs as-is (`= SOME(ARRAY[...])`, timestamp casts, `LIKE .. ESCAPE`, `st_intersects(.., st_geomfromgeojson(..))`). +- `ST_AsMVT(t, 'layer' ORDER BY ..)` / `ST_AsMVTGeom(geom, BOX_2D, extent, buffer, clip)` are available. +- Geometry columns carry their CRS in the type (`GEOMETRY('EPSG:26915')`, `GEOMETRY('OGC:CRS84')`, or plain `GEOMETRY`); `ST_CRS` / `ST_SetCRS` exist. +- Comparing geometries in different CRS silently returns no match → filter literals must be reprojected/tagged. +- `SET geometry_always_xy = true` is required for lon/lat axis order; `ST_Transform(g, 'EPSG:a', 'EPSG:b')` (or a PROJ string) then works positionally. Use `EPSG:4326` (not `OGC:CRS84`) as output CRS. +- `ST_AsGeoJSON` returns a JSON string; `ST_Extent_Agg` returns a GEOMETRY; no `ST_EstimatedExtent`. +- Primary keys via `duckdb_constraints()`. + +To confirm at the start of implementation: MVT with TIMESTAMP/DATE/LIST/JSON/DECIMAL properties (cast to VARCHAR if unsupported), empty-tile result (NULL → `b""`), TIMESTAMPTZ values in Python with `SET TimeZone = 'UTC'`, TEMP views in `duckdb_views()` / `duckdb_columns()`, CRS reported by `ST_Read`, cql2 emission of `{"op": "st_transform", "args": [geom, "EPSG:4326", "EPSG:n"]}`. + +## Changes + +### Setup +- Rebase `feat/duckdb-backend` on `main`. +- `pyproject.toml`: `duckdb = ["duckdb>=1.4"]` optional extra; add `tipg[duckdb]` to the dev group; `uv sync`. + +### `tipg/collections.py` — two overridable hooks (PG output unchanged) +`cql_where()` hard-codes PG constructs (`st_setsrid(polygon, srid)` in `_s_intersects_bbox`, `st_transform(literal, srid)` in `_transform_cql_geom_literals`). Route them through methods on `Collection` whose defaults keep today's PG output: +```python +def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: # default: st_setsrid(geom, srid) +def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: # default: st_transform(geom, srid) +``` +`DuckCollection` overrides with `st_setcrs(geom, 'EPSG:n')` and `st_transform(geom, 'EPSG:4326', 'EPSG:n')`. +Move `_quote_ident` from `tipg/pg/collections.py` to `tipg/collections.py` (same quoting rules in DuckDB). + +### `tipg/duck/settings.py` (new) +- `DuckDBSettings` (`TIPG_DUCKDB_`): `database: str = ":memory:"`, `read_only: bool = True`, `threads: int | None`, `memory_limit: str | None`, `sources: dict[str, str] = {}` (collection id → path/glob; parquet via `read_parquet`, others via `ST_Read`). +- `DuckDBCatalogSettings` (`TIPG_DUCKDB_`): `schemas = ["main"]`, `tables`, `exclude_tables`, `only_spatial_tables = True`, `spatial_extent = True`, `datetime_extent = True`. + +### `tipg/duck/database.py` (new) +- `connect_to_db(app, *, settings=None)`: `duckdb.connect(database, read_only)`, `INSTALL/LOAD spatial`, `SET geometry_always_xy = true`, `SET TimeZone = 'UTC'`, threads / memory_limit, register each source as `CREATE TEMP VIEW "" AS SELECT * FROM ...`; store on `app.state.duckdb_connection`. +- `close_db_connection(app)`. + +### `tipg/duck/collections.py` (new) +`DuckCollection(Collection)`: +- `connection: duckdb.DuckDBPyConnection = Field(exclude=True, repr=False)` (`arbitrary_types_allowed`), plus the fully qualified relation name (`"db"."schema"."table"`). +- `features()` / `get_tile()`: same signatures and validation as `PgCollection` (`InvalidGeometryColumnName`, `InvalidLimit`, sort-column check), same SQL-builder structure (`_select`, `_select_mvt`, `_from`, `_sort_expressions`, `_where_clause`). Queries run in `starlette.concurrency.run_in_threadpool` on a per-call `self.connection.cursor()` (closed after); count + page share the cursor. +- Geometry output: `ST_Transform(g, 'EPSG:', 'EPSG:4326')` when srid ≠ 4326; `ST_Envelope` for bbox-only; simplify = `ST_ReducePrecision(ST_SimplifyPreserveTopology(g, s), s)`; GeoJSON via `ST_AsGeoJSON` → `orjson.loads`; WKT = `'SRID=4326;' || ST_AsText(g)`; JSON-typed property values decoded with `orjson.loads`. +- Id: primary-key column, else `row_number() OVER ()`. +- MVT: inner `SELECT props, ST_AsMVTGeom(, ST_Extent(ST_MakeEnvelope()), res, buffer, clip) AS geom ... WHERE .. ORDER BY .. LIMIT n`; outer `SELECT ST_AsMVT(t, $1 [ORDER BY ..]) FROM t`. Out-of-domain tiles clipped with `ST_Intersection(ST_MakeEnvelope(tms.bbox), ST_Transform(g, src, 'EPSG:'))` like PG; TMS CRS as `EPSG:n` or PROJ string; NULL tile → `b""`. +- No function collections: `type` is `Table` / `View`; `function_parameters` accepted and ignored. + +`async duck_get_collection_index(connection, settings=None) -> list[Collection]` (blocking work in the threadpool): +- Tables/views of `current_database()` in `settings.schemas` (+ include/exclude filters), plus the TEMP source views. Ids: `schema.table`; sources use their key. +- Columns from `duckdb_columns()`, types normalized to the names `Column` already understands (`GEOMETRY*→geometry`, `TIMESTAMP→timestamp`, `TIMESTAMP WITH TIME ZONE→timestamptz`, `DATE→date`, integer types, `DOUBLE→float8`, `FLOAT→real`, `DECIMAL(..)→numeric`, `BOOLEAN→boolean`, `JSON→json`, `X[]→x[]`, else `text`) so `json_type`, `is_geometry`, `is_datetime` and `_coerce_id` work unchanged. +- Geometry: srid parsed from `GEOMETRY('EPSG:n')` (`OGC:CRS84` / no CRS → 4326); when `spatial_extent`, one scan per table for `ST_Extent_Agg` (→ 4326 if needed) and `min/max(ST_GeometryType)` (single type, else `"Geometry"`). +- Datetime extents (when `datetime_extent`): one `min/max` query per table covering all datetime columns, ISO strings matching PG output (naive timestamps as UTC, dates as midnight UTC). +- id / geometry / datetime column selection with `TableConfig` / `TableSettings` from `tipg.settings` (`fallback_key_names`, `sort_columns`, `table_config` keyed `_`, sources keyed by id), same logic as `pg_get_collection_index`. +- `only_spatial_tables` filter. + +### `tipg/duck/__init__.py` +Export `connect_to_db`, `close_db_connection`, `duck_get_collection_index`, `DuckCollection`, `DuckDBSettings`, `DuckDBCatalogSettings`. + +### Docs / changelog +- `docs/src/api/tipg/duck/{collections,database,settings}.md` + mkdocs nav. +- User-guide page: a DuckDB-only app and a mixed PG + DuckDB app (`register_collection_catalog` merging both index lists). +- `CHANGES.md` entries. + +## Tests (`tests/duck/`) +- `conftest.py`: session-scoped temp `.duckdb` built via SQL — points (INTEGER PK, TIMESTAMPTZ, DATE, text / double / list props), polygons without PK (`gid` fallback key), a `GEOMETRY('EPSG:26915')` table, a non-spatial table — plus a GeoParquet file declared as a source. App fixture: FastAPI + `Endpoints` + lifespan with a local `register_collection_catalog` (pattern of `tests/conftest.py:create_tipg_app`). A second fixture builds a mixed PG + DuckDB app. +- Coverage: `/collections` (extents, bbox / datetime filters), `/queryables`, `/items` with `ids`, `bbox`, `datetime`, cql2 `filter` (`IN`, `LIKE`, `S_INTERSECTS`, including on the EPSG:26915 table), `sortby`, `properties`, `limit` / `offset` + links, csv / ndjson (WKT path), `/items/{id}` (bad int id → 404), MVT (feature count, layer name, sortby order, empty tile), and the mixed app serving both a PG and a DuckDB collection. + +## Verification +1. Confirm the open DuckDB points listed above; adjust casts if needed. +2. `uv run pytest tests/duck -v`, then the full `uv run pytest` (PG suite guards the `cql_where` hook refactor). +3. `uv run pre-commit run --all-files`. +4. Manual: small app over the test `.duckdb` with uvicorn; check `/collections//items?f=html` and the tile viewer. + +## Implementation notes (deviations from the plan above) +- **Open points confirmed**: cql2 emits `st_transform(g, 'EPSG:4326', 'EPSG:n')` / `st_setcrs(g, 'EPSG:n')` as expected; `ST_Read` reports the CRS in the column type; an empty tile is a valid MVT with an empty layer (not NULL). +- **`pytz`**: DuckDB needs it to return `TIMESTAMP WITH TIME ZONE` values to Python → added to the `duckdb` extra. +- **Connection layout**: `connection.cursor()` opens a new connection, which can't see `TEMP` views or session-level `SET`. The connection now always uses an in-memory primary database; `database` (optional file path/URL) is `ATTACH`ed read-only under its file name (no `read_only` setting), sources are regular views in `memory.main`, and settings use `SET GLOBAL`. +- **Source ids**: sources are exposed as `main.` (same `schema.table` rule as tables) instead of their bare key; duplicate ids across attached databases are skipped with a warning. +- **MVT property casts**: `ST_AsMVT` supports BOOLEAN, integer types (not HUGEINT), FLOAT/DOUBLE, VARCHAR and JSON; DECIMAL/HUGEINT/UBIGINT are cast to DOUBLE, other types (dates, timestamps, lists, UUID, …) to VARCHAR. +- **Tests**: `tests/duck/conftest.py` imports `tipg.factory` inside fixtures; a module-level import read the module-level settings before the PG fixtures set their environment and broke 4 PG tests. diff --git a/pyproject.toml b/pyproject.toml index 106a00b9..177633f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "folium", "bump-my-version", "tipg[metrics]", + "tipg[duckdb]", ] server = [ "uvicorn[standard]>=0.12.0", @@ -69,6 +70,12 @@ docs = [ metrics = [ "prometheus-fastapi-instrumentator>=8.0.2", ] +duckdb = [ + # >=2.0: CRS aware GEOMETRY type, `ST_SetCRS`, `geometry_always_xy` + "duckdb>=2.0", + # required by duckdb to return TIMESTAMP WITH TIME ZONE values to Python + "pytz", +] [project.urls] Homepage = "https://developmentseed.org/tipg" diff --git a/tests/duck/__init__.py b/tests/duck/__init__.py new file mode 100644 index 00000000..31e137eb --- /dev/null +++ b/tests/duck/__init__.py @@ -0,0 +1 @@ +"""DuckDB backend tests.""" diff --git a/tests/duck/conftest.py b/tests/duck/conftest.py new file mode 100644 index 00000000..85ffdb1d --- /dev/null +++ b/tests/duck/conftest.py @@ -0,0 +1,206 @@ +"""DuckDB backend test fixtures.""" + +import datetime +from contextlib import asynccontextmanager + +import pytest +from fastapi import FastAPI +from starlette.requests import Request +from starlette.testclient import TestClient + +duckdb = pytest.importorskip("duckdb") + +from tipg.collections import Catalog # noqa: E402 +from tipg.duck import ( # noqa: E402 + DuckDBCatalogSettings, + DuckDBSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) + +FIXTURE_SQL = """ +-- 20 points on a diagonal, integer primary key, datetimes and various property types +CREATE TABLE points ( + id INTEGER PRIMARY KEY, + name VARCHAR, + value DOUBLE, + price DECIMAL(10, 2), + dt TIMESTAMPTZ, + day DATE, + tags VARCHAR[], + props JSON, + rank TINYINT, + code USMALLINT, + count UINTEGER, + geom GEOMETRY +); +INSERT INTO points +SELECT + x::INTEGER, + 'point ' || x, + x * 1.5, + x * 2.25, + TIMESTAMPTZ '2020-01-01 00:00:00+00' + to_hours(x::INTEGER), + DATE '2020-01-01' + x::INTEGER, + ['a', 'b'], + '{"rank": ' || x || '}', + x, + x * 100, + x * 1000000, + ST_Point(x, x) +FROM range(1, 21) r(x); + +-- polygons, no primary key (`gid` fallback key) +CREATE TABLE polygons AS +SELECT x::INTEGER AS gid, 'polygon ' || x AS name, ST_Buffer(ST_Point(x * 10, x * 10), 1) AS geom +FROM range(1, 4) r(x); + +-- geometries in a projected CRS (UTM zone 15N, around Minneapolis) +CREATE TABLE utm (id INTEGER PRIMARY KEY, name VARCHAR, geom GEOMETRY('EPSG:26915')); +INSERT INTO utm VALUES + (1, 'minneapolis', ST_SetCRS(ST_Point(478000, 4980000), 'EPSG:26915')), + (2, 'duluth', ST_SetCRS(ST_Point(568000, 5182000), 'EPSG:26915')); + +-- no geometry +CREATE TABLE nongeo AS SELECT x::INTEGER AS id, 'row ' || x AS name FROM range(1, 4) r(x); +""" + + +@pytest.fixture(scope="session") +def duckdb_files(tmp_path_factory): + """Create a DuckDB database file and a GeoParquet file.""" + tmp = tmp_path_factory.mktemp("duckdb") + database = str(tmp / "test.duckdb") + parquet = str(tmp / "landmarks.parquet") + + conn = duckdb.connect(database) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + conn.execute(FIXTURE_SQL) + conn.execute( + f"COPY (SELECT id, name, geom FROM points WHERE id <= 5) TO '{parquet}' (FORMAT parquet)" + ) + conn.close() + + return {"database": database, "parquet": parquet} + + +@pytest.fixture +def duckdb_settings(duckdb_files): + """DuckDB settings for the test database.""" + return DuckDBSettings( + database=duckdb_files["database"], + sources={"landmarks": duckdb_files["parquet"]}, + ) + + +def create_duck_app( + settings: DuckDBSettings, + catalog_settings: DuckDBCatalogSettings, +) -> FastAPI: + """Create a TiPg application serving DuckDB collections.""" + # NOTE: imported here (not at module level) so module-level settings are + # read after the environment set by the PostgreSQL test fixtures. + from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers + from tipg.factory import Endpoints + + async def register_collection_catalog(app: FastAPI) -> None: + collections = await duck_get_collection_index( + connection=app.state.duckdb_connection, + settings=catalog_settings, + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in collections}, + last_updated=datetime.datetime.now(), + ) + + @asynccontextmanager + async def lifespan(app: FastAPI): + await connect_to_duckdb(app, settings=settings) + await register_collection_catalog(app) + yield + await close_duckdb_connection(app) + + app = FastAPI(lifespan=lifespan) + app.include_router(Endpoints(title="TiPg DuckDB").router) + add_exception_handlers(app, DEFAULT_STATUS_CODES) + + @app.get("/rawcatalog") + async def raw_catalog(request: Request): + """Return parsed catalog data for testing.""" + return request.app.state.collection_catalog + + return app + + +@pytest.fixture +def duck_app(duckdb_settings): + """DuckDB-only TiPg app (all tables, spatial or not).""" + app = create_duck_app( + duckdb_settings, DuckDBCatalogSettings(only_spatial_tables=False) + ) + with TestClient(app) as client: + yield client + + +@pytest.fixture +def mixed_app(database_url, duckdb_settings): + """TiPg app serving both PostgreSQL and DuckDB collections.""" + from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers + from tipg.factory import Endpoints + from tipg.pg import ( + PostgresCatalogSettings, + PostgresSettings, + close_db_connection, + connect_to_db, + pg_get_collection_index, + ) + + postgres_catalog_settings = PostgresCatalogSettings(schemas=["public"]) + duckdb_catalog_settings = DuckDBCatalogSettings(tables=["main.points"]) + + @asynccontextmanager + async def lifespan(app: FastAPI): + await connect_to_db( + app, + schemas=postgres_catalog_settings.schemas, + tipg_schema=postgres_catalog_settings.tipg_schema, + settings=PostgresSettings(database_url=database_url), + ) + await connect_to_duckdb(app, settings=duckdb_settings) + + pg_collections = await pg_get_collection_index( + connection_pool=app.state.pool, + settings=postgres_catalog_settings, + ) + duck_collections = await duck_get_collection_index( + connection=app.state.duckdb_connection, + settings=duckdb_catalog_settings, + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*pg_collections, *duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + yield + + await close_duckdb_connection(app) + await close_db_connection(app) + + app = FastAPI(lifespan=lifespan) + app.include_router(Endpoints(title="TiPg mixed").router) + add_exception_handlers(app, DEFAULT_STATUS_CODES) + + with TestClient(app) as client: + yield client + + +@pytest.fixture +def duck_app_sources_only(duckdb_files): + """DuckDB TiPg app without database file (file sources only).""" + settings = DuckDBSettings(sources={"landmarks": duckdb_files["parquet"]}) + assert settings.database is None + app = create_duck_app(settings, DuckDBCatalogSettings()) + with TestClient(app) as client: + yield client diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py new file mode 100644 index 00000000..d9e71ed1 --- /dev/null +++ b/tests/duck/test_duck.py @@ -0,0 +1,343 @@ +"""Test the DuckDB backend through the OGC API endpoints.""" + +import json + +import mapbox_vector_tile +import pytest +from geojson_pydantic import Feature, FeatureCollection + +from tipg.duck import DuckCollection + + +def test_collections(duck_app): + """List DuckDB tables, views and file sources.""" + response = duck_app.get("/collections") + assert response.status_code == 200 + ids = {c["id"] for c in response.json()["collections"]} + assert ids == { + "main.points", + "main.polygons", + "main.utm", + "main.nongeo", + "main.landmarks", + } + + catalog = duck_app.app.state.collection_catalog["collections"] + assert all(isinstance(c, DuckCollection) for c in catalog.values()) + assert catalog["main.landmarks"].type == "View" + assert catalog["main.points"].type == "Table" + + # the connection is not serialized + response = duck_app.get("/rawcatalog") + assert response.status_code == 200 + assert "connection" not in response.json()["collections"]["main.points"] + + +def test_collection_metadata(duck_app): + """Check extents, CRS and queryables.""" + # tables are not scanned: no extents, generic geometry type + body = duck_app.get("/collections/main.points").json() + assert body["extent"]["spatial"]["bbox"] == [[-180.0, -90.0, 180.0, 90.0]] + assert "temporal" not in body["extent"] + catalog = duck_app.app.state.collection_catalog["collections"] + assert catalog["main.points"].geometry_column.bounds == [-180, -90, 180, 90] + assert catalog["main.points"].geometry_column.geometry_type == "Geometry" + assert catalog["main.points"].datetime_column.mindt is None + + # parquet source: extent and geometry type from the GeoParquet metadata + body = duck_app.get("/collections/main.landmarks").json() + assert body["extent"]["spatial"]["bbox"] == [[1.0, 1.0, 5.0, 5.0]] + assert catalog["main.landmarks"].geometry_column.geometry_type == "Point" + + utm = catalog["main.utm"] + assert utm.geometry_column.srid == 26915 + assert utm.crs == "http://www.opengis.net/def/crs/EPSG/0/26915" + + body = duck_app.get("/collections/main.nongeo").json() + assert not body.get("extent") + + body = duck_app.get("/collections/main.points/queryables").json() + props = body["properties"] + assert props["geom"] == {"$ref": "https://geojson.org/schema/Geometry.json"} + assert props["id"]["type"] == "number" + assert props["price"]["type"] == "number" + assert props["name"]["type"] == "string" + assert props["tags"]["type"] == "array" + assert props["props"]["type"] == "object" + + +def test_collections_filters(duck_app): + """Filter collections by bbox / datetime (uses collection extents).""" + body = duck_app.get("/collections", params={"bbox": "0,0,5,5"}).json() + ids = {c["id"] for c in body["collections"]} + # tables have no extent (world bbox), so they always match + assert {"main.points", "main.landmarks", "main.utm"} <= ids + + # landmarks extent (GeoParquet metadata) is [1, 1, 5, 5] + body = duck_app.get("/collections", params={"bbox": "10,10,20,20"}).json() + ids = {c["id"] for c in body["collections"]} + assert "main.landmarks" not in ids + assert "main.points" in ids + + body = duck_app.get( + "/collections", params={"datetime": "2019-01-01T00:00:00Z/2019-12-31T00:00:00Z"} + ).json() + assert body["numberMatched"] == 0 + + +def test_items(duck_app): + """Get items.""" + response = duck_app.get("/collections/main.points/items") + assert response.status_code == 200 + body = response.json() + FeatureCollection.model_validate(body) + assert body["numberMatched"] == 20 + assert body["numberReturned"] == 10 + feat = body["features"][0] + assert feat["id"] == 1 + assert feat["geometry"] == {"type": "Point", "coordinates": [1.0, 1.0]} + assert feat["properties"]["name"] == "point 1" + assert feat["properties"]["tags"] == ["a", "b"] + assert feat["properties"]["props"] == {"rank": 1} + assert feat["properties"]["dt"] == "2020-01-01T01:00:00+00:00" + assert "geom" not in feat["properties"] + + # paging + body = duck_app.get( + "/collections/main.points/items", params={"limit": 5, "offset": 15} + ).json() + assert [f["id"] for f in body["features"]] == [16, 17, 18, 19, 20] + rels = {link["rel"] for link in body["links"]} + assert "prev" in rels + assert "next" not in rels + + # no primary key: `gid` fallback key + body = duck_app.get("/collections/main.polygons/items").json() + assert [f["id"] for f in body["features"]] == [1, 2, 3] + + # file source + body = duck_app.get("/collections/main.landmarks/items").json() + assert body["numberMatched"] == 5 + + # non spatial table + body = duck_app.get("/collections/main.nongeo/items").json() + assert body["numberMatched"] == 3 + assert body["features"][0]["geometry"] is None + + +def test_items_filters(duck_app): + """Filter items with ids / bbox / datetime / properties / cql2.""" + url = "/collections/main.points/items" + + body = duck_app.get(url, params={"ids": "2,4"}).json() + assert [f["id"] for f in body["features"]] == [2, 4] + + body = duck_app.get(url, params={"bbox": "0,0,3.5,3.5"}).json() + assert [f["id"] for f in body["features"]] == [1, 2, 3] + + body = duck_app.get( + url, + params={"datetime": "2020-01-01T03:00:00Z/..", "datetime-column": "dt"}, + ).json() + # filtered requests don't count the matched features (full scan) + assert body.get("numberMatched") is None + assert [f["id"] for f in body["features"]] == list(range(3, 13)) + assert "next" in {link["rel"] for link in body["links"]} + + body = duck_app.get( + url, + params={ + "datetime": "2020-01-01T03:00:00Z/..", + "datetime-column": "dt", + "offset": 10, + }, + ).json() + assert [f["id"] for f in body["features"]] == list(range(13, 21)) + assert "next" not in {link["rel"] for link in body["links"]} + + # exactly `limit` features left: no next link + body = duck_app.get(url, params={"ids": "2,4", "limit": 2}).json() + assert len(body["features"]) == 2 + assert "next" not in {link["rel"] for link in body["links"]} + + body = duck_app.get(url, params={"name": "point 3"}).json() + assert [f["id"] for f in body["features"]] == [3] + + body = duck_app.get( + url, params={"filter": "id IN (5, 6, 7) AND name LIKE 'point %'"} + ).json() + assert [f["id"] for f in body["features"]] == [5, 6, 7] + + body = duck_app.get( + url, + params={ + "filter": "S_INTERSECTS(geom, POLYGON((0 0, 2.5 0, 2.5 2.5, 0 2.5, 0 0)))" + }, + ).json() + assert [f["id"] for f in body["features"]] == [1, 2] + + +def test_items_projected(duck_app): + """Filter and output items stored in EPSG:26915.""" + url = "/collections/main.utm/items" + + body = duck_app.get(url).json() + lon, lat = body["features"][0]["geometry"]["coordinates"] + assert -94 < lon < -93 and 44 < lat < 45.5 + + # bbox around Minneapolis only (CRS84) + body = duck_app.get(url, params={"bbox": "-94,44.5,-93,45.5"}).json() + assert [f["id"] for f in body["features"]] == [1] + + # cql2 geometries are CRS84 too + body = duck_app.get( + url, + params={ + "filter": "S_INTERSECTS(geom, POLYGON((-93 46, -91 46, -91 47.5, -93 47.5, -93 46)))" + }, + ).json() + assert [f["id"] for f in body["features"]] == [2] + + +def test_items_sortby_properties(duck_app): + """Sort and select properties.""" + url = "/collections/main.points/items" + + body = duck_app.get(url, params={"sortby": "-value", "limit": 3}).json() + assert [f["id"] for f in body["features"]] == [20, 19, 18] + + body = duck_app.get(url, params={"properties": "name", "limit": 1}).json() + assert body["features"][0]["properties"] == {"name": "point 1"} + + response = duck_app.get(url, params={"sortby": "nope"}) + assert response.status_code == 404 + + +def test_items_output_formats(duck_app): + """CSV / NDJSON outputs use the WKT geometry path.""" + url = "/collections/main.points/items" + + response = duck_app.get(url, params={"f": "csv", "limit": 2}) + assert response.status_code == 200 + rows = response.text.splitlines() + assert len(rows) == 3 + assert "SRID=4326;POINT (1 1)" in rows[1] + + response = duck_app.get(url, params={"f": "ndjson", "limit": 2}) + assert response.status_code == 200 + first = json.loads(response.text.splitlines()[0]) + assert first["geometry"] == "SRID=4326;POINT (1 1)" + + response = duck_app.get(url, params={"f": "geojsonseq", "limit": 2}) + assert response.status_code == 200 + Feature.model_validate(json.loads(response.text.splitlines()[0])) + + response = duck_app.get(url, params={"f": "html", "limit": 2}) + assert response.status_code == 200 + + +def test_item(duck_app): + """Get one item.""" + response = duck_app.get("/collections/main.points/items/3") + assert response.status_code == 200 + assert response.json()["id"] == 3 + + response = duck_app.get("/collections/main.points/items/999") + assert response.status_code == 404 + + # non-integer id on an integer key + response = duck_app.get("/collections/main.points/items/abc") + assert response.status_code == 404 + + +@pytest.mark.parametrize( + "collection,count", + [("main.points", 20), ("main.polygons", 3), ("main.utm", 2), ("main.landmarks", 5)], +) +def test_tiles(duck_app, collection, count): + """Get vector tiles.""" + response = duck_app.get(f"/collections/{collection}/tiles/WebMercatorQuad/0/0/0") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/vnd.mapbox-vector-tile" + decoded = mapbox_vector_tile.decode(response.content) + assert len(decoded["default"]["features"]) == count + + +def test_tiles_options(duck_app): + """Vector tiles properties, sortby, limit and empty tiles.""" + url = "/collections/main.points/tiles/WebMercatorQuad/0/0/0" + + decoded = mapbox_vector_tile.decode(duck_app.get(url).content) + props = decoded["default"]["features"][0]["properties"] + # types not supported by MVT are cast + assert props["price"] == 2.25 + assert props["day"] == "2020-01-02" + assert props["tags"] == "[a, b]" + # integer types ST_AsMVT rejects (TINYINT, USMALLINT, UINTEGER) are cast + assert props["rank"] == 1 + assert props["code"] == 100 + assert props["count"] == 1000000 + + response = duck_app.get( + url, params={"sortby": "-id", "limit": 3, "properties": "id"} + ) + decoded = mapbox_vector_tile.decode(response.content) + assert [f["properties"]["id"] for f in decoded["default"]["features"]] == [ + 20, + 19, + 18, + ] + + response = duck_app.get(url, params={"sortby": "value", "properties": "id"}) + assert response.status_code == 404 + + # tile without data + response = duck_app.get("/collections/main.points/tiles/WebMercatorQuad/10/0/0") + assert response.status_code == 200 + decoded = mapbox_vector_tile.decode(response.content) + assert not decoded or not decoded["default"]["features"] + + +def test_mixed_catalog(mixed_app): + """Serve PostgreSQL and DuckDB collections from one catalog.""" + ids = {c["id"] for c in mixed_app.get("/collections").json()["collections"]} + assert "main.points" in ids + assert "public.landsat_wrs" in ids + + body = mixed_app.get("/collections/main.points/items", params={"limit": 1}).json() + assert body["numberMatched"] == 20 + + body = mixed_app.get( + "/collections/public.landsat_wrs/items", params={"limit": 1} + ).json() + assert body["numberMatched"] == 16269 + + response = mixed_app.get("/collections/main.points/tiles/WebMercatorQuad/0/0/0") + assert response.status_code == 200 + response = mixed_app.get( + "/collections/public.landsat_wrs/tiles/WebMercatorQuad/0/0/0" + ) + assert response.status_code == 200 + + +def test_sources_only(duck_app_sources_only): + """Without database file, only the file sources are served.""" + body = duck_app_sources_only.get("/collections").json() + assert [c["id"] for c in body["collections"]] == ["main.landmarks"] + + body = duck_app_sources_only.get("/collections/main.landmarks/items").json() + assert body["numberMatched"] == 5 + + +def test_database_alias(duckdb_files): + """The database file is attached under its file name, also for URLs.""" + from tipg.duck import DuckDBSettings, create_connection + + conn = create_connection( + DuckDBSettings(database=f"file://{duckdb_files['database']}") + ) + rows = conn.execute( + "SELECT database_name, readonly FROM duckdb_databases() WHERE NOT internal" + ).fetchall() + assert ("test", True) in rows + conn.close() diff --git a/tests/duck/test_geoparquet.py b/tests/duck/test_geoparquet.py new file mode 100644 index 00000000..8904f009 --- /dev/null +++ b/tests/duck/test_geoparquet.py @@ -0,0 +1,543 @@ +"""Test GeoParquet metadata (bbox, geometry types) used by the DuckDB catalog.""" + +import asyncio +import json + +import mapbox_vector_tile +import morecantile +import pytest +from cql2 import Expr +from pyproj import CRS, Transformer + +from tipg.collections import crs_uri +from tipg.duck import DuckDBCatalogSettings, DuckDBSettings +from tipg.duck.collections import ( + _collection_index, + _geometry_srid, + _geoparquet_metadata, + _literal_bounds, + _parquet_view_source, + _parse_covering, +) +from tipg.duck.database import create_connection + +duckdb = pytest.importorskip("duckdb") + + +@pytest.fixture(scope="module") +def parquet_dir(tmp_path_factory): + """Write GeoParquet files (DuckDB writes the `geo` metadata).""" + tmp = tmp_path_factory.mktemp("geoparquet") + conn = duckdb.connect() + conn.execute("LOAD spatial") + conn.execute( + f"COPY (SELECT 1 AS id, ST_Point(1, 2) AS geom) TO '{tmp}/part-a.parquet'" + ) + conn.execute( + f"COPY (SELECT 2 AS id, ST_Buffer(ST_Point(10, 20), 1) AS geom) TO '{tmp}/part-b.parquet'" + ) + conn.execute(f"COPY (SELECT 1 AS id, 'a' AS name) TO '{tmp}/nongeo.parquet'") + # no id column, `value` not in insertion order + conn.execute( + f"COPY (SELECT 'f' || x AS name, 100 - x AS value, ST_Point(x, x) AS geom " + f"FROM range(1, 26) r(x)) TO '{tmp}/noid.parquet'" + ) + conn.execute( + f"COPY (SELECT 42 AS file_row_number, ST_Point(0, 0) AS geom) TO '{tmp}/frn.parquet'" + ) + + # GeoParquet 1.1 with a bbox covering column (5 row groups, points on a + # diagonal from 0 to 99.99): DuckDB doesn't write coverings, so the + # geometry is written as WKB with our own `geo` metadata + (tmp / "covering").mkdir() + geo = json.dumps( + { + "version": "1.1.0", + "primary_column": "geom", + "columns": { + "geom": { + "encoding": "WKB", + "geometry_types": ["Point"], + "bbox": [0, 0, 99.99, 99.99], + "covering": { + "bbox": { + "xmin": ["bbox", "xmin"], + "ymin": ["bbox", "ymin"], + "xmax": ["bbox", "xmax"], + "ymax": ["bbox", "ymax"], + } + }, + } + }, + } + ) + conn.execute( + f""" + COPY ( + SELECT + x AS fid, + ST_AsWKB(ST_Point(x / 100, x / 100)) AS geom, + {{'xmin': x / 100, 'ymin': x / 100, 'xmax': x / 100, 'ymax': x / 100}} AS bbox + FROM range(0, 10000) r(x) + ) TO '{tmp}/covering/points.parquet' + (FORMAT parquet, ROW_GROUP_SIZE 2048, KV_METADATA {{geo: '{geo}'}}) + """ + ) + + # Same covering metadata, but the struct fields don't match it (`min_x` + # instead of `xmin`, like some source.coop datasets) + (tmp / "badcovering").mkdir() + conn.execute( + f""" + COPY ( + SELECT + x AS fid, + ST_AsWKB(ST_Point(x, x)) AS geom, + {{'min_x': x, 'min_y': x, 'max_x': x, 'max_y': x}} AS bbox + FROM range(0, 10) r(x) + ) TO '{tmp}/badcovering/points.parquet' + (FORMAT parquet, KV_METADATA {{geo: '{geo}'}}) + """ + ) + # covering metadata pointing to a struct with unknown field names + (tmp / "invalidcovering").mkdir() + conn.execute( + f""" + COPY ( + SELECT + x AS fid, + ST_AsWKB(ST_Point(x, x)) AS geom, + {{'a': x, 'b': x, 'c': x, 'd': x}} AS bbox + FROM range(0, 10) r(x) + ) TO '{tmp}/invalidcovering/points.parquet' + (FORMAT parquet, KV_METADATA {{geo: '{geo}'}}) + """ + ) + + # GeoParquet with a PROJJSON CRS without authority code (custom Lambert + # projection, like the Austria Lambert of some fiboa datasets) + (tmp / "projjson").mkdir() + crs = CRS.from_proj4(CUSTOM_LCC) + to_crs = Transformer.from_crs(4326, crs, always_xy=True) + points = [to_crs.transform(lon, lat) for lon, lat in LONLAT_POINTS] + xs, ys = [p[0] for p in points], [p[1] for p in points] + geo = json.dumps( + { + "version": "1.1.0", + "primary_column": "geom", + "columns": { + "geom": { + "encoding": "WKB", + "geometry_types": ["Point"], + "crs": crs.to_json_dict(), + "bbox": [min(xs), min(ys), max(xs), max(ys)], + } + }, + } + ) + values = ", ".join( + f"({i}, ST_AsWKB(ST_Point({x}, {y})))" for i, (x, y) in enumerate(points) + ) + conn.execute( + f""" + COPY (SELECT * FROM (VALUES {values}) t(fid, geom)) + TO '{tmp}/projjson/points.parquet' + (FORMAT parquet, KV_METADATA {{geo: '{geo}'}}) + """ + ) + + conn.close() + return tmp + + +# Custom Lambert Conformal Conic projection (no EPSG code) +CUSTOM_LCC = ( + "+proj=lcc +lat_0=47.5 +lon_0=13.3333333333333 +lat_1=49 +lat_2=46 " + "+x_0=400000 +y_0=400000 +ellps=GRS80 +units=m +no_defs" +) +# (lon, lat) of the points of the PROJJSON CRS file +LONLAT_POINTS = [(13.0, 47.0), (13.5, 47.5), (14.0, 48.0), (16.0, 48.2)] + + +def test_parquet_view_source(): + """Only plain `SELECT * FROM read_parquet(...)` views are recognized.""" + assert _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('s3://b/it''s/*.parquet');" + ) == ("s3://b/it's/*.parquet", False) + # as stored by DuckDB for `read_parquet('x.parquet', file_row_number = true)` + assert _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('x.parquet', (file_row_number = true));" + ) == ("x.parquet", True) + assert _parquet_view_source(None) is None + assert ( + _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('x.parquet') WHERE id > 1;" + ) + is None + ) + assert ( + _parquet_view_source("CREATE VIEW a AS SELECT * FROM ST_Read('x.shp');") is None + ) + + +def test_geoparquet_metadata(parquet_dir): + """Read and merge bbox/geometry types from the file footers.""" + conn = duckdb.connect() + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-a.parquet") + assert meta == { + "geom": { + "bbox": [1.0, 2.0, 1.0, 2.0], + "geometry_types": {"Point"}, + "covering": None, + } + } + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/covering/points.parquet") + assert meta["geom"]["covering"] == { + "xmin": '"bbox"."xmin"', + "ymin": '"bbox"."ymin"', + "xmax": '"bbox"."xmax"', + "ymax": '"bbox"."ymax"', + } + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-*.parquet") + assert meta["geom"]["bbox"] == [1.0, 2.0, 11.0, 21.0] + assert meta["geom"]["geometry_types"] == {"Point", "Polygon"} + + # no `geo` metadata + assert _geoparquet_metadata(conn, f"{parquet_dir}/nongeo.parquet") == {} + # mixed files: some files have no `geo` metadata + assert _geoparquet_metadata(conn, f"{parquet_dir}/*.parquet") == {} + # errors fall back to an empty dict + assert _geoparquet_metadata(conn, f"{parquet_dir}/missing.parquet") == {} + + +def test_catalog_uses_geoparquet_metadata(parquet_dir): + """Collection extent and geometry type come from the GeoParquet metadata.""" + conn = create_connection( + DuckDBSettings( + sources={ + "single": f"{parquet_dir}/part-a.parquet", + "multi": f"{parquet_dir}/part-*.parquet", + } + ) + ) + try: + collections = { + c.id: c for c in _collection_index(conn, DuckDBCatalogSettings()) + } + finally: + conn.close() + + single = collections["main.single"].geometry_column + assert single.bounds == [1.0, 2.0, 1.0, 2.0] + assert single.geometry_type == "Point" + + multi = collections["main.multi"].geometry_column + assert multi.bounds == [1.0, 2.0, 11.0, 21.0] + assert multi.geometry_type == "Geometry" + + +def test_features_without_id_column(parquet_dir): + """No default sort, ids are row positions for collections without primary key.""" + conn = create_connection( + # a glob: no `file_row_number` id (row numbers are not unique across files) + DuckDBSettings(sources={"noid": f"{parquet_dir}/noid*.parquet"}) + ) + try: + (collection,) = _collection_index(conn, DuckDBCatalogSettings()) + assert collection.id_column is None + # no ORDER BY without `sortby` (would be a full scan) + assert collection._sortby(None) == "" + + items = asyncio.run(collection.features(limit=10, offset=10)) + assert items["matched"] == 25 + assert [f["id"] for f in items["items"]] == list(range(11, 21)) + # insertion order + assert [f["properties"]["name"] for f in items["items"]] == [ + f"f{x}" for x in range(11, 21) + ] + + # explicit sort: ids are positions in the sorted features + items = asyncio.run(collection.features(sortby="value", limit=5)) + assert [f["id"] for f in items["items"]] == [1, 2, 3, 4, 5] + assert [f["properties"]["value"] for f in items["items"]] == [ + 75, + 76, + 77, + 78, + 79, + ] + finally: + conn.close() + + +def test_file_row_number_id(parquet_dir): + """Single parquet file sources use the row position in the file as id.""" + conn = create_connection( + DuckDBSettings( + sources={ + "noid": f"{parquet_dir}/noid.parquet", + # already has a `file_row_number` column + "frn": f"{parquet_dir}/frn.parquet", + } + ) + ) + try: + collections = { + c.id: c for c in _collection_index(conn, DuckDBCatalogSettings()) + } + col = collections["main.noid"] + assert col.id_column.name == "file_row_number" + + # ids are stable: same id for a feature with or without filter/sort + items = asyncio.run(col.features(limit=3)) + assert [f["id"] for f in items["items"]] == [0, 1, 2] + assert [f["properties"]["name"] for f in items["items"]] == ["f1", "f2", "f3"] + + items = asyncio.run(col.features(cql_filter=Expr("value < 80"), limit=2)) + assert [f["id"] for f in items["items"]] == [20, 21] + assert [f["properties"]["name"] for f in items["items"]] == ["f21", "f22"] + + items = asyncio.run(col.features(sortby="-name", limit=1)) + assert [f["id"] for f in items["items"]] == [8] # "f9" + + # item lookup + items = asyncio.run(col.features(ids_filter=["20"])) + assert [f["properties"]["name"] for f in items["items"]] == ["f21"] + + # existing `file_row_number` column: plain view, regular column + col = collections["main.frn"] + assert col.get_column("file_row_number") + items = asyncio.run(col.features()) + assert items["items"][0]["properties"]["file_row_number"] == 42 + finally: + conn.close() + + +def test_literal_bounds(): + """Bounds of cql2-json geometry literals.""" + point = {"type": "Point", "coordinates": [1, 2]} + polygon = { + "type": "Polygon", + "coordinates": [[[0, 0], [2, 0], [2, 3], [0, 3], [0, 0]]], + } + assert _literal_bounds(point, 4326) == [1.0, 2.0, 1.0, 2.0] + assert _literal_bounds(polygon, 4326) == [0.0, 0.0, 2.0, 3.0] + assert _literal_bounds({"bbox": [0, 1, 2, 3]}, 4326) == [0.0, 1.0, 2.0, 3.0] + assert _literal_bounds( + {"type": "GeometryCollection", "geometries": [point, polygon]}, 4326 + ) == [0.0, 0.0, 2.0, 3.0] + # tagged with the column CRS: coordinates are in the column CRS + tagged = {"op": "st_setcrs", "args": [polygon, "EPSG:26915"]} + assert _literal_bounds(tagged, 26915) == [0.0, 0.0, 2.0, 3.0] + # untagged literal for a projected column, or reprojected literal: ignored + assert _literal_bounds(polygon, 26915) is None + transformed = {"op": "st_transform", "args": [polygon, "EPSG:4326", "EPSG:26915"]} + assert _literal_bounds(transformed, 26915) is None + assert _literal_bounds({"property": "geom"}, 4326) is None + + +@pytest.fixture +def covering_collection(parquet_dir): + """Collection of a GeoParquet file with a bbox covering column.""" + conn = create_connection( + DuckDBSettings(sources={"cov": f"{parquet_dir}/covering/points.parquet"}) + ) + (collection,) = _collection_index(conn, DuckDBCatalogSettings()) + yield collection + conn.close() + + +def _row_groups_scanned(collection, where_clause: str) -> str: + """`Row Groups Scanned: x / y` from the query profile.""" + plan = collection.connection.execute( + f"EXPLAIN ANALYZE SELECT count(*) {collection._from()} {where_clause}" + ).fetchall()[0][1] + (line,) = [line for line in plan.splitlines() if "Row Groups Scanned" in line] + return line.strip("│ ").split(":")[1].strip() + + +def test_bbox_covering_filter(covering_collection): + """Spatial filters also filter the bbox covering columns (row group pruning).""" + col = covering_collection + assert col.bbox_coverings == { + "geom": { + "xmin": '"bbox"."xmin"', + "ymin": '"bbox"."ymin"', + "xmax": '"bbox"."xmax"', + "ymax": '"bbox"."ymax"', + } + } + assert col.geometry_column.geometry_type == "Point" + + # no spatial filter + assert col._where_clause(None) == "WHERE TRUE" + where = col.cql_where(cql=Expr("fid > 10")) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + + # bbox filter + where = col.cql_where(bbox=[12, 12, 15, 15]) + clause = col._where_clause(where) + assert clause.startswith( + 'WHERE "bbox"."xmin" <= 15.0 AND "bbox"."xmax" >= 12.0 ' + 'AND "bbox"."ymin" <= 15.0 AND "bbox"."ymax" >= 12.0 AND (' + ) + assert _row_groups_scanned(col, clause) == "1 / 5" + assert _row_groups_scanned(col, f"WHERE {where.to_sql()}") == "5 / 5" + + items = asyncio.run(col.features(bbox_filter=[12, 12, 15, 15], limit=1000)) + assert len(items["items"]) == 301 + assert items["matched"] is None + + # cql2 spatial filter, combined with other filters + where = col.cql_where( + cql=Expr( + "fid > 10 AND S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))" + ) + ) + assert _row_groups_scanned(col, col._where_clause(where)) == "1 / 5" + + # spatial filter under OR/NOT: the covering filter can't be applied + where = col.cql_where( + cql=Expr( + "fid = 1 OR S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))" + ) + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + where = col.cql_where( + cql=Expr("NOT S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))") + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + where = col.cql_where( + cql=Expr("S_DISJOINT(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))") + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + + +def test_bbox_covering_tile(covering_collection): + """Tile envelope filter also filters the bbox covering columns.""" + col = covering_collection + tms = morecantile.tms.get("WebMercatorQuad") + tile = tms.tile(13.5, 13.5, 6) + where = col.cql_where(tms=tms, tile=tile) + assert _row_groups_scanned(col, col._where_clause(where)) == "1 / 5" + + content = asyncio.run(col.get_tile(tms=tms, tile=tile)) + assert content + + +@pytest.mark.parametrize("source", ["badcovering", "invalidcovering"]) +def test_invalid_bbox_covering(parquet_dir, source): + """Sources with a covering not matching the struct fields are excluded.""" + conn = create_connection( + DuckDBSettings( + sources={ + "bad": f"{parquet_dir}/{source}/points.parquet", + "good": f"{parquet_dir}/covering/points.parquet", + } + ) + ) + try: + collections = _collection_index(conn, DuckDBCatalogSettings()) + assert [c.id for c in collections] == ["main.good"] + finally: + conn.close() + + +def test_parse_covering(): + """Missing covering → None, malformed covering → empty dict.""" + assert _parse_covering({}) is None + assert _parse_covering({"covering": {"bbox": "bbox"}}) == {} + assert _parse_covering({"covering": {"bbox": {"xmin": ["bbox", "xmin"]}}}) == {} + assert _parse_covering( + { + "covering": { + "bbox": {k: ["bbox", k] for k in ("xmin", "ymin", "xmax", "ymax")} + } + } + ) == {k: f'"bbox"."{k}"' for k in ("xmin", "ymin", "xmax", "ymax")} + + +def test_geometry_srid(): + """Geometry column CRS: EPSG code, or CRS definition.""" + assert _geometry_srid("GEOMETRY") == 4326 + assert _geometry_srid("GEOMETRY('EPSG:26915')") == 26915 + assert _geometry_srid("GEOMETRY('OGC:CRS84')") == 4326 + + # PROJJSON of an EPSG CRS + assert _geometry_srid(f"GEOMETRY('{CRS.from_epsg(2154).to_json()}')") == 2154 + assert ( + _geometry_srid(f"GEOMETRY('{CRS.from_user_input('OGC:CRS84').to_json()}')") + == 4326 + ) + + # CRS without EPSG code: kept as is + ignf = CRS.from_user_input("IGNF:LAMB93").to_json() + assert _geometry_srid(f"GEOMETRY('{ignf}')") == ignf + assert crs_uri(ignf) == "http://www.opengis.net/def/crs/IGNF/0/LAMB93" + + custom = CRS.from_proj4(CUSTOM_LCC).to_json() + assert _geometry_srid(f"GEOMETRY('{custom}')") == custom + assert crs_uri(custom) is None + + # unknown CRS: assumed EPSG:4326 + assert _geometry_srid("GEOMETRY('FOO:BAR')") == 4326 + + assert crs_uri(4326) == "http://www.opengis.net/def/crs/EPSG/0/4326" + + +@pytest.fixture +def projjson_collection(parquet_dir): + """Collection of a GeoParquet file with a PROJJSON CRS (no EPSG code).""" + conn = create_connection( + DuckDBSettings(sources={"lcc": f"{parquet_dir}/projjson/points.parquet"}) + ) + (collection,) = _collection_index(conn, DuckDBCatalogSettings()) + yield collection + conn.close() + + +def test_projjson_crs(projjson_collection): + """Serve geometries stored in a CRS without EPSG code.""" + col = projjson_collection + geom = col.geometry_column + assert isinstance(geom.srid, str) + assert CRS.from_user_input(geom.srid) == CRS.from_proj4(CUSTOM_LCC) + assert col.crs is None + + # extent reprojected to WGS84 + minx, miny, maxx, maxy = geom.bounds + assert minx == pytest.approx(13.0, abs=0.1) + assert maxx == pytest.approx(16.0, abs=0.1) + assert miny == pytest.approx(47.0, abs=0.1) + assert maxy == pytest.approx(48.2, abs=0.1) + + # geometries reprojected to WGS84 + items = asyncio.run(col.features()) + coords = [f["geometry"]["coordinates"] for f in items["items"]] + assert coords == [ + [pytest.approx(lon, abs=1e-6), pytest.approx(lat, abs=1e-6)] + for lon, lat in LONLAT_POINTS + ] + + # bbox filter (WGS84) + items = asyncio.run(col.features(bbox_filter=[12.9, 46.9, 13.6, 47.6])) + assert [f["id"] for f in items["items"]] == [0, 1] + + # cql2 spatial filter (WGS84 literal) + items = asyncio.run( + col.features( + cql_filter=Expr( + "S_INTERSECTS(geom, POLYGON((13.9 47.9, 16.1 47.9, 16.1 48.3, 13.9 48.3, 13.9 47.9)))" + ) + ) + ) + assert [f["id"] for f in items["items"]] == [2, 3] + + # tile + tms = morecantile.tms.get("WebMercatorQuad") + content = asyncio.run(col.get_tile(tms=tms, tile=tms.tile(13.5, 47.5, 8))) + decoded = mapbox_vector_tile.decode(content) + assert len(decoded["default"]["features"]) == 1 diff --git a/tipg/collections.py b/tipg/collections.py index 107b4f4f..b636cde1 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -10,7 +10,8 @@ from cql2 import Expr from morecantile import Tile, TileMatrixSet from pydantic import BaseModel, model_validator -from pyproj import Transformer +from pyproj import CRS, Transformer +from pyproj.exceptions import CRSError from .errors import ( InvalidDatetime, @@ -23,8 +24,31 @@ TransformerFromCRS = lru_cache(Transformer.from_crs) +# CRS of a geometry column: an EPSG code, or a CRS definition understood by +# pyproj and the database (e.g PROJJSON or `AUTH:CODE`) for CRS without EPSG +# code (only produced by the DuckDB backend). +Srid = int | str + # Wraps a cql2-json geometry node into a backend-specific function call -GeometryLiteralWrapper = Callable[[Any, int], Any] +GeometryLiteralWrapper = Callable[[Any, Srid], Any] + + +@lru_cache(maxsize=256) +def crs_uri(srid: Srid) -> str | None: + """OGC URI of a CRS (``None`` for CRS definitions without authority code).""" + if isinstance(srid, int): + return f"http://www.opengis.net/def/crs/EPSG/0/{srid}" + + try: + authority = CRS.from_user_input(srid).to_authority(min_confidence=100) + except CRSError: + return None + + if authority is None: + return None + + auth, code = authority + return f"http://www.opengis.net/def/crs/{auth}/0/{code}" def _quote_ident(name: str) -> str: @@ -70,7 +94,7 @@ def _s_intersects_bbox( south: float, east: float, north: float, - srid: int | None = None, + srid: Srid | None = None, *, tag: GeometryLiteralWrapper, ) -> Expr: @@ -113,7 +137,7 @@ def _s_intersects_bbox( def _transform_cql_geom_literals( node: Any, - target_srid: int, + target_srid: Srid, reproject: GeometryLiteralWrapper, ) -> Any: """Walk a user-supplied cql2-json tree and wrap every GeoJSON geometry @@ -180,7 +204,8 @@ class Column(BaseModel): type: str description: str | None = None geometry_type: str | None = None - srid: int | None = None + # EPSG code, or CRS definition (see `Srid`) + srid: Srid | None = None bounds: list[float] | None = None mindt: str | None = None maxdt: str | None = None @@ -319,8 +344,8 @@ def dt_bounds(self) -> list[str] | None: @property def crs(self): """Return crs of set geometry column.""" - if self.geometry_column: - return f"http://www.opengis.net/def/crs/EPSG/0/{self.geometry_column.srid}" + if self.geometry_column and self.geometry_column.srid is not None: + return crs_uri(self.geometry_column.srid) @cached_property def geometry_columns(self) -> list[Column]: @@ -401,14 +426,14 @@ def queryables(self) -> dict: return {**geoms, **props} @abc.abstractmethod - def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Tag a cql2-json geometry literal, whose coordinates are already in ``srid``, with that CRS (returns a cql2-json function call node). """ ... @abc.abstractmethod - def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Reproject a EPSG:4326 cql2-json geometry literal to ``srid`` (returns a cql2-json function call node). """ diff --git a/tipg/duck/__init__.py b/tipg/duck/__init__.py new file mode 100644 index 00000000..43d04249 --- /dev/null +++ b/tipg/duck/__init__.py @@ -0,0 +1,25 @@ +"""tipg.duck package: DuckDB integration for TiPg.""" + +import warnings + +from ..errors import TipgExperimentalWarning +from .collections import DuckCollection, duck_get_collection_index +from .database import close_duckdb_connection, connect_to_duckdb, create_connection +from .settings import DuckDBCatalogSettings, DuckDBSettings + +__all__ = [ + "DuckCollection", + "duck_get_collection_index", + "close_duckdb_connection", + "connect_to_duckdb", + "create_connection", + "DuckDBCatalogSettings", + "DuckDBSettings", +] + + +warnings.warn( + "This module is experimental, its contents are subject to change and deprecation.", + category=TipgExperimentalWarning, + stacklevel=2, +) diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py new file mode 100644 index 00000000..6cba9f9b --- /dev/null +++ b/tipg/duck/collections.py @@ -0,0 +1,1188 @@ +"""tipg DuckDB collection models.""" + +import functools +import math +import re +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +import duckdb +import orjson +from cql2 import Expr +from morecantile import Tile, TileMatrixSet +from pydantic import ConfigDict, Field +from pyproj import CRS +from pyproj.exceptions import CRSError +from starlette.concurrency import run_in_threadpool + +from ..collections import Collection, Column, Feature, ItemList, Srid, _quote_ident +from ..errors import InvalidGeometryColumnName, InvalidLimit, InvalidPropertyName +from ..logger import logger +from ..settings import FeaturesSettings, MVTSettings, TableConfig, TableSettings +from .settings import DuckDBCatalogSettings + +mvt_settings = MVTSettings() +features_settings = FeaturesSettings() # type: ignore [call-arg] + +# Column types (as normalized in the catalog) ST_AsMVT can encode as-is. +# NOTE: `text` is not one of them: unknown DuckDB types (e.g STRUCT, MAP) are +# normalized to `text`, so they must be cast to VARCHAR. +_MVT_NATIVE_TYPES = frozenset({"real", "float8", "boolean", "json"}) + +# Integer types (as normalized in the catalog) → DuckDB type ST_AsMVT accepts. +# The normalized name may come from a type ST_AsMVT rejects (e.g. ``UINTEGER`` +# is normalized to ``bigint``, ``TINYINT`` to ``smallint``), so always cast. +_MVT_INTEGER_CASTS = { + "smallint": "INTEGER", + "integer": "INTEGER", + "bigint": "BIGINT", +} + + +# cql2 spatial predicates which imply that the bboxes of the geometries intersect +_BBOX_PREDICATES = frozenset( + { + "s_intersects", + "s_equals", + "s_touches", + "s_within", + "s_overlaps", + "s_crosses", + "s_contains", + } +) + + +def _conjuncts(expr: Any) -> Iterator[Any]: + """Terms of the top-level ``AND`` of a cql2-json expression.""" + if isinstance(expr, dict) and expr.get("op") == "and": + for arg in expr.get("args") or []: + yield from _conjuncts(arg) + else: + yield expr + + +def _coords_bounds(coords: Any) -> list[float] | None: + """[xmin, ymin, xmax, ymax] of (nested) GeoJSON coordinates.""" + if not isinstance(coords, list) or not coords: + return None + + # a position + if all(isinstance(v, (int, float)) for v in coords): + if len(coords) < 2: + return None + x, y = float(coords[0]), float(coords[1]) + return [x, y, x, y] + + bounds = [_coords_bounds(c) for c in coords] + if any(b is None for b in bounds): + return None + + return [ + min(b[0] for b in bounds), # type: ignore [index] + min(b[1] for b in bounds), # type: ignore [index] + max(b[2] for b in bounds), # type: ignore [index] + max(b[3] for b in bounds), # type: ignore [index] + ] + + +def _literal_bounds(literal: Any, srid: Srid) -> list[float] | None: + """Bounds of a cql2-json geometry literal, in the column CRS (``srid``). + + Literals of columns not in EPSG:4326 are expected to be tagged with their + CRS (``st_setcrs``, see ``DuckCollection._tag_geometry_literal``), other + ones (e.g reprojected with ``st_transform``) are ignored. + """ + if isinstance(literal, dict) and literal.get("op") == "st_setcrs": + literal = (literal.get("args") or [None])[0] + elif srid != 4326: + return None + + if not isinstance(literal, dict): + return None + + if bbox := literal.get("bbox"): + if len(bbox) == 6: + bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] + bounds = [float(v) for v in bbox] if len(bbox) == 4 else None + + elif literal.get("type") == "GeometryCollection": + parts = [_literal_bounds(g, 4326) for g in literal.get("geometries") or []] + if not parts or any(p is None for p in parts): + return None + bounds = _coords_bounds( + [[p[0], p[1]] for p in parts] + [[p[2], p[3]] for p in parts] + ) # type: ignore [index] + + else: + bounds = _coords_bounds(literal.get("coordinates")) + + if bounds is None or not all(math.isfinite(v) for v in bounds): + return None + + return bounds + + +def _quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def _crs(srid: Srid) -> str: + """DuckDB CRS identifier for an EPSG code (or a CRS definition, as is).""" + if isinstance(srid, str): + return srid + return f"EPSG:{srid}" + + +def _crs_literal(srid: Srid) -> str: + """Quoted DuckDB CRS identifier (SQL string literal).""" + return _quote_literal(_crs(srid)) + + +class DuckCollection(Collection): + """Model for DuckDB Table and View.""" + + database: str + dbschema: str = Field(alias="schema") + + # Reference to the (shared) DuckDB connection serving this collection. + # Excluded from serialization and repr. + connection: duckdb.DuckDBPyConnection = Field(exclude=True, repr=False) + + # GeoParquet bbox covering of geometry columns: column name → SQL + # expression of its ``xmin``, ``ymin``, ``xmax`` and ``ymax`` columns. + bbox_coverings: dict[str, dict[str, str]] = {} + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @property + def _qualified_name(self) -> str: + """SQL-quoted ``"database"."schema"."table"`` identifier for this collection.""" + return ".".join( + _quote_ident(p) for p in (self.database, self.dbschema, self.table) + ) + + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: + """Tag a geometry literal with its CRS (DuckDB ``ST_SetCRS``).""" + return {"op": "st_setcrs", "args": [geom, _crs(srid)]} + + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: + """Reproject a EPSG:4326 geometry literal to ``srid`` (DuckDB ``ST_Transform``).""" + return {"op": "st_transform", "args": [geom, _crs(4326), _crs(srid)]} + + def _select_property_columns(self, properties: list[str] | None) -> list[str]: + """Quoted column-name fragments for the property columns to SELECT.""" + return [_quote_ident(c) for c in self.columns(properties)] + + def _select_mvt_property_columns(self, properties: list[str] | None) -> list[str]: + """Property columns to SELECT for MVT encoding. + + ``ST_AsMVT`` only supports boolean, integer, float and string values: + decimals are cast to DOUBLE and other types to VARCHAR. + """ + cols = [] + for name in self.columns(properties): + col = self.get_column(name) + q = _quote_ident(name) + if col is None or col.type in _MVT_NATIVE_TYPES: + cols.append(q) + elif cast := _MVT_INTEGER_CASTS.get(col.type): + cols.append(f"CAST({q} AS {cast}) AS {q}") + elif col.type == "numeric": + cols.append(f"CAST({q} AS DOUBLE) AS {q}") + else: + cols.append(f"CAST({q} AS VARCHAR) AS {q}") + return cols + + def _select_id(self) -> str | None: + """SQL fragment that selects the primary-key column as ``tipg_id``. + + Collections without primary key get their ids from the row position + (see ``_run_features_queries``): a ``ROW_NUMBER()`` window would + prevent DuckDB from skipping rows (``OFFSET``) cheaply. + """ + if self.id_column: + return f"{_quote_ident(self.id_column.name)} AS tipg_id" + return None + + def _geom_expr( + self, + geometry_column: Column | None, + bbox_only: bool | None, + simplify: float | None, + ) -> str | None: + """Build the SQL expression for the geometry column (reprojected to + 4326 if needed, optionally bbox-only or simplified). + """ + if geometry_column is None: + return None + + g = _quote_ident(geometry_column.name) + + if geometry_column.srid != 4326: + g = f"ST_Transform({g}, {_crs_literal(geometry_column.srid)}, {_crs_literal(4326)})" # type: ignore [arg-type] + + if bbox_only: + g = f"ST_Envelope({g})" + elif simplify: + s = float(simplify) + g = f"ST_ReducePrecision(ST_SimplifyPreserveTopology({g}, {s}), {s})" + + return g + + def _select( + self, + properties: list[str] | None, + geometry_column: Column | None, + bbox_only: bool | None, + simplify: float | None, + geom_as_wkt: bool = False, + ) -> str: + """Build the SELECT clause for the main features query.""" + cols = self._select_property_columns(properties) + if select_id := self._select_id(): + cols.append(select_id) + + geom = self._geom_expr(geometry_column, bbox_only, simplify) + if geom_as_wkt: + cols.append( + f"'SRID=4326;' || ST_AsText({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS VARCHAR) AS tipg_geom" + ) + else: + cols.append( + f"ST_AsGeoJSON({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS VARCHAR) AS tipg_geom" + ) + + return "SELECT " + ", ".join(cols) + + def _select_mvt( + self, + properties: list[str] | None, + geometry_column: Column, + tms: TileMatrixSet, + tile: Tile, + simplify: float | None = None, + ) -> str: + """Build the SELECT clause that emits an MVT geometry per row.""" + cols = self._select_mvt_property_columns(properties) + + geom = _quote_ident(geometry_column.name) + src_crs = _crs(geometry_column.srid) # type: ignore [arg-type] + + if simplify: + geom = f"ST_ReducePrecision(ST_SimplifyPreserveTopology({geom}, {simplify}), {simplify})" + + # For tiles outside the TMS's natural domain, clip the geometry to the + # TMS's geographic bbox before reprojecting (see PgCollection._select_mvt). + if not tms.is_valid(tile, strict=False): + west, south, east, north = tms.bbox + geo_crs = _crs(tms.geographic_crs.to_epsg() or 4326) + geom = ( + f"ST_Intersection(" + f"ST_MakeEnvelope({west}, {south}, {east}, {north}), " + f"ST_Transform({geom}, {_quote_literal(src_crs)}, {_quote_literal(geo_crs)})" + f")" + ) + src_crs = geo_crs + + # Reproject to TMS CRS — prefer EPSG code, fall back to PROJ string. + if tms_srid := tms.crs.to_epsg(): + tms_crs = _crs(tms_srid) + else: + tms_crs = tms.crs.to_proj4() + + transformed = f"ST_Transform({geom}, {_quote_literal(src_crs)}, {_quote_literal(tms_crs)})" + + bbox = tms.xy_bounds(tile) + envelope = f"ST_Extent(ST_MakeEnvelope({bbox.left}, {bbox.bottom}, {bbox.right}, {bbox.top}))" + cols.append( + f"ST_AsMVTGeom(" + f"{transformed}, {envelope}, " + f"{int(mvt_settings.tile_resolution)}, " + f"{int(mvt_settings.tile_buffer)}, " + f"{'TRUE' if mvt_settings.tile_clip else 'FALSE'}" + f") AS geom" + ) + + return "SELECT " + ", ".join(cols) + + def _from(self) -> str: + """Build the FROM clause.""" + return f"FROM {self._qualified_name}" + + def _sort_expressions(self, sortby: str | None) -> list[tuple[str, str]]: + """Parse ``sortby`` into ``(column name, SQL fragment)`` pairs. + + Without ``sortby``, no sort is applied: sorting means a full scan of + the sort column, and DuckDB preserves the insertion order + (``preserve_insertion_order``), so paging stays stable without it. + """ + sorts: list[tuple[str, str]] = [] + if not sortby: + return sorts + + for s in sortby.strip().split(","): + parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore + + direction = parts["direction"] + column = parts["column"].strip() + if not self.get_column(column): + raise InvalidPropertyName(f"Property {column} does not exist.") + col_sql = _quote_ident(column) + sorts.append((column, f"{col_sql} DESC" if direction == "-" else col_sql)) + + return sorts + + def _sortby(self, sortby: str | None) -> str: + """Build the ORDER BY clause (empty without ``sortby``).""" + if sorts := self._sort_expressions(sortby): + return "ORDER BY " + ", ".join(sql for _, sql in sorts) + return "" + + def _bbox_covering_filter(self, where: Expr) -> str | None: + """SQL filter on the GeoParquet bbox covering columns. + + For each spatial predicate (e.g the `bbox` or tile filters) between a + geometry column with a bbox covering and a geometry literal, which is + part of the top-level ``AND`` of ``where``, filter the covering + columns with the bounds of the literal. DuckDB uses the row group + statistics of those columns to skip reading most of the file. + """ + if not self.bbox_coverings: + return None + + filters = [] + for expr in _conjuncts(where.to_json()): + if not isinstance(expr, dict) or expr.get("op") not in _BBOX_PREDICATES: + continue + + args = expr.get("args") or [] + if len(args) != 2: + continue + + for prop, literal in (args, args[::-1]): + if not isinstance(prop, dict) or "property" not in prop: + continue + + name = prop["property"] + covering = self.bbox_coverings.get(name) + column = self.get_column(name) + if not covering or not column: + continue + + if bounds := _literal_bounds(literal, column.srid or 4326): + xmin, ymin, xmax, ymax = bounds + filters.append( + f"{covering['xmin']} <= {xmax!r} AND {covering['xmax']} >= {xmin!r} " + f"AND {covering['ymin']} <= {ymax!r} AND {covering['ymax']} >= {ymin!r}" + ) + break + + return " AND ".join(filters) or None + + def _where_clause(self, where: Expr | None) -> str: + """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" + if where is None: + return "WHERE TRUE" + + sql = where.to_sql() + if covering := self._bbox_covering_filter(where): + sql = f"{covering} AND ({sql})" + + return f"WHERE {sql}" + + def _row_to_feature( + self, row: dict[str, Any], geom_as_wkt: bool, position: int + ) -> Feature: + """Convert a result row to a Feature. + + ``position`` (1-based row position in the matched features) is used as + id for collections without primary key. + """ + g = row.pop("tipg_geom") + id = row.pop("tipg_id", position) + if g is not None and not geom_as_wkt: + g = orjson.loads(g) + + # JSON columns are returned as strings + for name, value in row.items(): + if isinstance(value, str) and (col := self.get_column(name)): + if col.type == "json": + row[name] = orjson.loads(value) + + return Feature(type="Feature", geometry=g, id=id, properties=row) + + def _run_features_queries( + self, + count_sql: str | None, + sql: str, + geom_as_wkt: bool, + offset: int, + limit: int, + ) -> tuple[int | None, list[Feature], bool]: + """Run the COUNT (if any) and Features queries (blocking). + + ``sql`` must select up to ``limit + 1`` rows: the extra row tells if + there are more features. Returns ``(matched, features, has_more)``. + """ + cursor = self.connection.cursor() + try: + matched = None + if count_sql: + logger.debug(count_sql) + matched = cursor.execute(count_sql).fetchone()[0] # type: ignore [index] + + logger.debug(sql) + cursor.execute(sql) + names = [d[0] for d in cursor.description] # type: ignore [union-attr] + rows = cursor.fetchall() + features = [ + self._row_to_feature(dict(zip(names, r)), geom_as_wkt, offset + i) + for i, r in enumerate(rows[:limit], start=1) + ] + finally: + cursor.close() + + return matched, features, len(rows) > limit + + def _run_tile_query(self, sql: str, layer: str) -> bytes: + """Run the MVT query (blocking).""" + cursor = self.connection.cursor() + try: + logger.debug(sql) + tile = cursor.execute(sql, [layer]).fetchone()[0] # type: ignore [index] + finally: + cursor.close() + + return bytes(tile) if tile is not None else b"" + + async def features( + self, + *, + ids_filter: list[str] | None = None, + bbox_filter: list[float] | None = None, + datetime_filter: list[str] | None = None, + properties_filter: list[tuple[str, str]] | None = None, + cql_filter: Expr | None = None, + sortby: str | None = None, + properties: list[str] | None = None, + geom: str | None = None, + dt: str | None = None, + limit: int | None = None, + offset: int | None = None, + bbox_only: bool | None = None, + simplify: float | None = None, + geom_as_wkt: bool = False, + **kwargs: Any, + ) -> ItemList: + """Build and run DuckDB query.""" + limit = limit or features_settings.default_features_limit + offset = offset or 0 + + if geom and geom.lower() != "none" and not self.get_geometry_column(geom): + raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") + + if limit and limit > features_settings.max_features_per_query: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" + ) + + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + ) + where = self._where_clause(where_filter) + + # Without filter, DuckDB answers COUNT(*) from the table/file metadata. + # With a filter it is a full scan (while the features query can stop + # after `limit` rows), so `numberMatched` is not returned. + count_sql = None + if where_filter is None: + count_sql = " ".join(["SELECT COUNT(*)", self._from()]) + + sql = " ".join( + [ + self._select( + properties=properties, + geometry_column=self.get_geometry_column(geom), + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + ), + self._from(), + where, + self._sortby(sortby), + f"LIMIT {int(limit) + 1}", + f"OFFSET {int(offset)}", + ] + ) + + matched, features, has_more = await run_in_threadpool( + self._run_features_queries, count_sql, sql, geom_as_wkt, offset, limit + ) + returned = len(features) + + return ItemList( + items=features, + matched=matched, + next=offset + returned if has_more else None, + prev=max(offset - limit, 0) if offset else None, + ) + + async def get_tile( + self, + *, + tms: TileMatrixSet, + tile: Tile, + ids_filter: list[str] | None = None, + bbox_filter: list[float] | None = None, + datetime_filter: list[str] | None = None, + properties_filter: list[tuple[str, str]] | None = None, + cql_filter: Expr | None = None, + sortby: str | None = None, + properties: list[str] | None = None, + geom: str | None = None, + dt: str | None = None, + limit: int | None = None, + simplify: float | None = None, + **kwargs: Any, + ) -> bytes: + """Build query to get Vector Tile.""" + limit = limit or mvt_settings.max_features_per_tile + + geometry_column = self.get_geometry_column(geom) + if not geometry_column: + raise InvalidGeometryColumnName(f"Invalid Geometry Column Name {geom}") + + if limit > mvt_settings.max_features_per_tile: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_tile` setting of {mvt_settings.max_features_per_tile}" + ) + + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + tms=tms, + tile=tile, + ) + + sorts = self._sort_expressions(sortby) + order_by = self._sortby(sortby) + + inner_sql = " ".join( + [ + self._select_mvt( + properties=properties, + geometry_column=geometry_column, + tms=tms, + tile=tile, + simplify=simplify, + ), + self._from(), + self._where_clause(where_filter), + order_by, + f"LIMIT {int(limit)}", + ] + ) + + # Only an ORDER BY inside the aggregate guarantees the feature order + # in the tile. It can only reference columns of ``t``, so an explicit + # sort column must be projected. + aggregate_order = "" + if sortby: + selected = self.columns(properties) + if missing := [col for col, _ in sorts if col not in selected]: + raise InvalidPropertyName( + f"Cannot sort by {', '.join(missing)}: not in the selected `properties`." + ) + aggregate_order = f" {order_by}" + + layer = self.table if mvt_settings.set_mvt_layername is True else "default" + sql = f"WITH t AS ({inner_sql}) SELECT ST_AsMVT(t, $1{aggregate_order}) FROM t" + + return await run_in_threadpool(self._run_tile_query, sql, layer) + + +_DUCK_TYPES = { + "BOOLEAN": "boolean", + "TINYINT": "smallint", + "SMALLINT": "smallint", + "UTINYINT": "smallint", + "INTEGER": "integer", + "USMALLINT": "integer", + "BIGINT": "bigint", + "UINTEGER": "bigint", + "UBIGINT": "numeric", + "HUGEINT": "numeric", + "UHUGEINT": "numeric", + "FLOAT": "real", + "DOUBLE": "float8", + "DATE": "date", + "TIMESTAMP": "timestamp", + "TIMESTAMP_S": "timestamp", + "TIMESTAMP_MS": "timestamp", + "TIMESTAMP_NS": "timestamp", + "TIMESTAMP WITH TIME ZONE": "timestamptz", + "JSON": "json", +} + +# ST_GeometryType values → PostGIS-style geometry type names +_GEOMETRY_TYPES = { + "POINT": "Point", + "LINESTRING": "LineString", + "POLYGON": "Polygon", + "MULTIPOINT": "MultiPoint", + "MULTILINESTRING": "MultiLineString", + "MULTIPOLYGON": "MultiPolygon", + "GEOMETRYCOLLECTION": "GeometryCollection", +} + +_GEOMETRY_CRS = re.compile(r"^GEOMETRY\('(?P[^']+)'\)$", re.IGNORECASE) + + +def _normalize_type(data_type: str) -> str: + """Map a DuckDB type name to the type names `tipg.collections.Column` understands.""" + t = data_type.upper() + if t == "GEOMETRY" or _GEOMETRY_CRS.match(t): + return "geometry" + + if t.endswith("[]"): + return _normalize_type(data_type[:-2]) + "[]" + + if t.startswith("DECIMAL"): + return "numeric" + + return _DUCK_TYPES.get(t, "text") + + +@functools.lru_cache(maxsize=256) +def _parse_crs(crs: str) -> Srid | None: + """EPSG code of a CRS definition when it is exactly an EPSG CRS, else the + definition itself (``None`` if it can't be parsed by pyproj). + """ + try: + authority = CRS.from_user_input(crs).to_authority(min_confidence=100) + except CRSError: + return None + + if authority: + auth, code = authority + if auth == "EPSG" and code.isdigit(): + return int(code) + if (auth, code) == ("OGC", "CRS84"): + return 4326 + + return crs + + +def _geometry_srid(data_type: str) -> Srid: + """CRS of a geometry column, from its type (e.g `GEOMETRY('EPSG:26915')`). + + Returns the EPSG code (geometries without CRS, or in OGC:CRS84, are + assumed to be in EPSG:4326), or the CRS definition (e.g PROJJSON from + GeoParquet metadata, or `AUTH:CODE`) for CRS without EPSG code. + """ + if m := _GEOMETRY_CRS.match(data_type): + crs = m.group("crs") + auth, _, code = crs.partition(":") + if auth.upper() == "EPSG" and code.isdigit(): + return int(code) + + if crs.upper() in ("OGC:CRS84", "CRS84"): + return 4326 + + if (srid := _parse_crs(crs)) is not None: + return srid + + logger.warning(f"Unsupported geometry CRS {crs[:100]}, assuming EPSG:4326") + + return 4326 + + +_PARQUET_VIEW = re.compile( + r"^CREATE VIEW .+? AS SELECT \* FROM read_parquet\('(?P(?:[^']|'')*)'" + r"(?P, \(file_row_number = true\))?\);?$", + re.DOTALL | re.IGNORECASE, +) + +# Column added by `read_parquet(..., file_row_number = true)` +FILE_ROW_NUMBER = "file_row_number" + + +def _parquet_view_source(view_sql: str | None) -> tuple[str, bool] | None: + """Path (or glob) of a view defined as ``SELECT * FROM read_parquet('')`` + and whether the ``file_row_number`` option is set (as done for file sources, + see ``tipg.duck.database``). + """ + if view_sql and (m := _PARQUET_VIEW.match(view_sql.strip())): + return m.group("path").replace("''", "'"), bool(m.group("row_number")) + + return None + + +_COVERING_KEYS = ("xmin", "ymin", "xmax", "ymax") + + +def _parse_covering(meta: dict[str, Any]) -> dict[str, str] | None: + """SQL expressions of a GeoParquet bbox covering (e.g ``"bbox"."xmin"``). + + Returns ``None`` without bbox covering, and an empty dict for a malformed + one (rejected by ``_valid_covering``). + """ + covering = (meta.get("covering") or {}).get("bbox") + if covering is None: + return None + + if not isinstance(covering, dict): + return {} + + exprs = {} + for key in _COVERING_KEYS: + path = covering.get(key) + if ( + not path + or not isinstance(path, list) + or not all(isinstance(p, str) for p in path) + ): + return {} + exprs[key] = ".".join(_quote_ident(p) for p in path) + + return exprs + + +def _valid_covering( + cursor: duckdb.DuckDBPyConnection, + covering: dict[str, str], + column_types: dict[str, str], +) -> bool: + """Check that a bbox covering matches the actual column types. + + GeoParquet metadata can be wrong (e.g. a covering pointing to + ``bbox.xmin`` while the struct fields are ``min_x``...). The covering + expressions are bound against typed NULL values, so no data is read. + """ + if any(key not in covering for key in _COVERING_KEYS): + return False + + # only bind the columns the covering refers to (e.g. `"bbox"` for `"bbox"."xmin"`) + exprs = [covering[key] for key in _COVERING_KEYS] + referenced = { + name: column_types[name] for name in _covering_columns(covering, column_types) + } + if not referenced: + return False + + columns = ", ".join( + f"NULL::{dtype} AS {_quote_ident(name)}" for name, dtype in referenced.items() + ) + try: + cursor.execute(f"SELECT {', '.join(exprs)} FROM (SELECT {columns}) LIMIT 0") + except duckdb.Error: + return False + + return True + + +def _covering_columns( + covering: dict[str, str], + column_types: dict[str, str], +) -> list[str]: + """Names of the columns a bbox covering refers to.""" + exprs = [covering[key] for key in _COVERING_KEYS] + return [ + name + for name in column_types + if any(e == (q := _quote_ident(name)) or e.startswith(f"{q}.") for e in exprs) + ] + + +def _parse_geo_metadata(geo: str) -> dict[str, dict[str, Any]]: + """Bbox, geometry types and bbox covering per column, from a GeoParquet + ``geo`` metadata value (each one is ``None`` when unknown). + """ + columns: dict[str, dict[str, Any]] = {} + for name, meta in orjson.loads(geo).get("columns", {}).items(): + bbox = meta.get("bbox") + if not bbox or len(bbox) not in (4, 6): + bbox = None + # 3D bbox: [xmin, ymin, zmin, xmax, ymax, zmax] + elif len(bbox) == 6: + bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] + + types = meta.get("geometry_types") + columns[name] = { + "bbox": [float(v) for v in bbox] if bbox else None, + "geometry_types": set(types) if types and bbox else None, + "covering": _parse_covering(meta), + } + + return columns + + +def _merge_geo_metadata(col: dict[str, Any], other: dict[str, Any]) -> None: + """Merge the GeoParquet metadata of a column from another file (in place).""" + if col["bbox"] is None or other["bbox"] is None: + col["bbox"] = None + col["geometry_types"] = None + else: + col["bbox"] = [ + min(col["bbox"][0], other["bbox"][0]), + min(col["bbox"][1], other["bbox"][1]), + max(col["bbox"][2], other["bbox"][2]), + max(col["bbox"][3], other["bbox"][3]), + ] + if col["geometry_types"] is None or other["geometry_types"] is None: + col["geometry_types"] = None + else: + col["geometry_types"] |= other["geometry_types"] + + # a malformed covering (empty dict) in any file invalidates the source + if col["covering"] == {} or other["covering"] == {}: + col["covering"] = {} + elif col["covering"] != other["covering"]: + col["covering"] = None + + +def _geoparquet_metadata( + cursor: duckdb.DuckDBPyConnection, + path: str, +) -> dict[str, dict[str, Any]]: + """Bbox, geometry types and bbox covering of geometry columns, from + GeoParquet metadata. + + Only the file footers are read. For multiple files (glob), the metadata + is merged: a column is only returned when every file describes it, its + bbox (and geometry types) is ``None`` when one file has no bbox and its + covering is ``None`` when files have different coverings. + """ + try: + nfiles = cursor.execute("SELECT count(*) FROM glob($1)", [path]).fetchone()[0] # type: ignore [index] + rows = cursor.execute( + """ + SELECT max(decode(value)) FILTER (WHERE decode(key) = 'geo') + FROM parquet_kv_metadata($1) + GROUP BY file_name + """, + [path], + ).fetchall() + except duckdb.Error as e: + logger.debug(f"Could not read GeoParquet metadata of {path}: {e}") + return {} + + # files without any key/value metadata are not listed by `parquet_kv_metadata` + if not rows or len(rows) != nfiles or any(geo is None for (geo,) in rows): + return {} + + merged: dict[str, dict[str, Any]] | None = None + for (geo,) in rows: + file_columns = _parse_geo_metadata(geo) + if merged is None: + merged = file_columns + continue + + for name in list(merged): + if (other := file_columns.get(name)) is None: + del merged[name] + else: + _merge_geo_metadata(merged[name], other) + + return merged or {} + + +def _geoparquet_metadatas( + connection: duckdb.DuckDBPyConnection, + paths: set[str], + max_workers: int, +) -> dict[str, dict[str, dict[str, Any]]]: + """GeoParquet metadata of multiple parquet sources, read in parallel. + + Each worker uses its own cursor (DuckDB cursors can run queries + concurrently on the same database instance). + """ + + def _read(path: str) -> dict[str, dict[str, Any]]: + cursor = connection.cursor() + try: + return _geoparquet_metadata(cursor, path) + finally: + cursor.close() + + if not paths: + return {} + + paths_list = sorted(paths) + with ThreadPoolExecutor( + max_workers=max(1, min(max_workers, len(paths_list))) + ) as pool: + return dict(zip(paths_list, pool.map(_read, paths_list))) + + +def _apply_geo_metadata( + cursor: duckdb.DuckDBPyConnection, + column: dict[str, Any], + meta: dict[str, Any], +) -> None: + """Set a geometry column bounds (in EPSG:4326) and type from GeoParquet metadata.""" + xmin, ymin, xmax, ymax = meta["bbox"] + if column["srid"] != 4326: + env = f"ST_Transform(ST_MakeEnvelope($1, $2, $3, $4), {_crs_literal(column['srid'])}, {_crs_literal(4326)})" + xmin, ymin, xmax, ymax = cursor.execute( + f"SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e) FROM (SELECT {env} AS e)", + [xmin, ymin, xmax, ymax], + ).fetchone() # type: ignore [misc] + + column["bounds"] = [xmin, ymin, xmax, ymax] + + # e.g. "Polygon Z" -> "POLYGON" + if types := meta["geometry_types"]: + base_types = {t.split()[0].upper() for t in types} + if len(base_types) == 1: + column["geometry_type"] = _GEOMETRY_TYPES.get(base_types.pop(), "Geometry") + + +def _geometry_metadata( + cursor: duckdb.DuckDBPyConnection, + columns: list[dict[str, Any]], + geo_metadata: dict[str, dict[str, Any]], +) -> None: + """Add geometry type and bounds to the geometry columns (in place). + + Rows are never scanned: bounds and type only come from ``geo_metadata`` + (GeoParquet file metadata). Other geometry columns have no bounds and a + generic ``Geometry`` type. + """ + for c in columns: + if c["type"] == "geometry": + c["geometry_type"] = "Geometry" + if (meta := geo_metadata.get(c["name"])) and meta["bbox"]: + _apply_geo_metadata(cursor, c, meta) + + +def _collection_index( # noqa: C901 + connection: duckdb.DuckDBPyConnection, + settings: DuckDBCatalogSettings, +) -> list[Collection]: + """Fetch Tables and Views index (blocking).""" + cursor = connection.cursor() + try: + relations = cursor.execute( + """ + SELECT database_name, schema_name, table_name, 'Table', comment, NULL + FROM duckdb_tables() + WHERE NOT internal AND NOT temporary + UNION ALL + SELECT database_name, schema_name, view_name, 'View', comment, sql + FROM duckdb_views() + WHERE NOT internal AND NOT temporary + ORDER BY 1, 2, 3 + """ + ).fetchall() + + columns_rows = cursor.execute( + """ + SELECT database_name, schema_name, table_name, column_name, data_type, comment + FROM duckdb_columns() + WHERE NOT internal + ORDER BY database_name, schema_name, table_name, column_index + """ + ).fetchall() + + pk_rows = cursor.execute( + """ + SELECT database_name, schema_name, table_name, constraint_column_names + FROM duckdb_constraints() + WHERE constraint_type = 'PRIMARY KEY' + """ + ).fetchall() + + table_columns: dict[tuple[str, str, str], list[dict[str, Any]]] = {} + # raw DuckDB column types, used to validate GeoParquet bbox coverings + table_types: dict[tuple[str, str, str], dict[str, str]] = {} + for db, schema, table, name, data_type, comment in columns_rows: + table_types.setdefault((db, schema, table), {})[name] = data_type + col: dict[str, Any] = { + "name": name, + "type": _normalize_type(data_type), + "description": comment, + } + if col["type"] == "geometry": + col["srid"] = _geometry_srid(data_type) + table_columns.setdefault((db, schema, table), []).append(col) + + pks = { + (db, schema, table): names[0] + for db, schema, table, names in pk_rows + if len(names) == 1 + } + + table_settings = TableSettings() + table_confs = table_settings.table_config + fallback_key_names = table_settings.fallback_key_names + + # Select the relations to expose + selected: dict[str, tuple[Any, ...]] = {} + for db, schema, table, entity, comment, view_sql in relations: + if schema not in settings.schemas: + continue + + table_id = f"{schema}.{table}" + if settings.tables is not None and table_id not in settings.tables: + continue + + if settings.exclude_tables and table_id in settings.exclude_tables: + continue + + if table_id in selected: + logger.warning( + f"Collection {table_id} already registered, skipping `{db}`.{table_id}" + ) + continue + + columns = table_columns.get((db, schema, table), []) + if settings.only_spatial_tables and not any( + c["type"] == "geometry" for c in columns + ): + continue + + selected[table_id] = ( + db, + schema, + table, + entity, + comment, + columns, + _parquet_view_source(view_sql), + ) + + # Read the GeoParquet metadata of parquet views, in parallel (one + # footer request per file, which adds up for many remote files) + geo_metadatas: dict[str, dict[str, dict[str, Any]]] = {} + if settings.spatial_extent: + geo_metadatas = _geoparquet_metadatas( + connection, + {src[0] for *_, src in selected.values() if src}, + settings.metadata_threads, + ) + + collections: dict[str, Collection] = {} + for table_id, ( + db, + schema, + table, + entity, + comment, + columns, + parquet_source, + ) in selected.items(): + geo_metadata = ( + geo_metadatas.get(parquet_source[0], {}) if parquet_source else {} + ) + # GeoParquet bbox coverings: sources with a covering which doesn't + # follow the spec (e.g. pointing to missing struct fields) are excluded + bbox_coverings = { + name: meta["covering"] + for name, meta in geo_metadata.items() + if meta["covering"] is not None + } + types = table_types.get((db, schema, table), {}) + if invalid := [ + name + for name, covering in bbox_coverings.items() + if not _valid_covering(cursor, covering, types) + ]: + logger.warning( + f"Collection {table_id} excluded: invalid GeoParquet bbox " + f"covering for column(s) {invalid} " + f"({ {n: bbox_coverings[n] for n in invalid} })" + ) + continue + + _geometry_metadata(cursor, columns, geo_metadata) + + table_conf = table_confs.get(f"{schema}_{table}", TableConfig()) + + if table_settings.sort_columns: + columns = sorted(columns, key=lambda d: d["name"]) + + cols = [Column(**c) for c in columns] + properties_setting = table_conf.properties or [c.name for c in cols] + + # ID Column + id_column = None + if id_name := table_conf.pk or pks.get((db, schema, table)): + for column in cols: + if id_name == column.name: + id_column = column + break + + if id_column is None and fallback_key_names: + for column in cols: + if column.name in fallback_key_names: + id_column = column + break + + # Parquet file source: row position in the file + if id_column is None and parquet_source and parquet_source[1]: + for column in cols: + if column.name == FILE_ROW_NUMBER: + id_column = column + break + + datetime_column = None + geometry_column = None + for column in cols: + if column.is_datetime: + if table_conf.datetimecol == column.name or datetime_column is None: + datetime_column = column + + if column.is_geometry: + if table_conf.geomcol == column.name or geometry_column is None: + geometry_column = column + + collections[table_id] = DuckCollection( + type=entity, + id=table_id, + table=table, + database=db, + schema=schema, + description=comment, + table_columns=cols, + properties=[c for c in cols if c.name in properties_setting], + id_column=id_column, + datetime_column=datetime_column, + geometry_column=geometry_column, + bbox_coverings=bbox_coverings, + connection=connection, + ) + + finally: + cursor.close() + + return list(collections.values()) + + +async def duck_get_collection_index( + *, + connection: duckdb.DuckDBPyConnection, + settings: DuckDBCatalogSettings | None = None, +) -> list[Collection]: + """Fetch Tables and Views index.""" + return await run_in_threadpool( + _collection_index, connection, settings or DuckDBCatalogSettings() + ) diff --git a/tipg/duck/database.py b/tipg/duck/database.py new file mode 100644 index 00000000..da8d64a2 --- /dev/null +++ b/tipg/duck/database.py @@ -0,0 +1,117 @@ +"""tipg DuckDB database events.""" + +import pathlib +from urllib.parse import urlparse + +import duckdb +from fastapi import FastAPI + +from ..collections import _quote_ident +from ..logger import logger +from .settings import DuckDBSettings + + +def _quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def _is_parquet(path: str) -> bool: + """Is the source a parquet file (or glob).""" + return path.lower().endswith((".parquet", ".parq", ".geoparquet")) + + +def _is_glob(path: str) -> bool: + """Is the source a glob pattern (multiple files).""" + return any(c in path for c in ("*", "?", "[")) + + +def _source_query(path: str, file_row_number: bool = False) -> str: + """SQL `SELECT` for a file source. + + With `file_row_number`, parquet sources get a `file_row_number` column + (row position in the file), used as stable feature id. + """ + if _is_parquet(path): + options = ", file_row_number = true" if file_row_number else "" + return f"SELECT * FROM read_parquet({_quote_literal(path)}{options})" + + return f"SELECT * FROM ST_Read({_quote_literal(path)})" + + +def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: + """Create and configure a DuckDB connection. + + The connection always uses an in-memory primary database, where file + `sources` are registered as views. When `settings.database` is set, the + database file is attached, read-only, under its file name + (without extension). + + Note: views (and settings) must be visible from `connection.cursor()`, + which creates a new connection to the same database instance, so we + can't use TEMP views (connection-local) nor session-level `SET`. + """ + config: dict[str, str | bool | int | float | list[str]] = {} + if settings.threads: + config["threads"] = settings.threads + if settings.memory_limit: + config["memory_limit"] = settings.memory_limit + + conn = duckdb.connect(":memory:", config=config) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + + # DuckDB GEOMETRY uses [EASTING, NORTHING] (lon/lat) order internally, + # make sure ST_Transform and GDAL follow it. + conn.execute("SET GLOBAL geometry_always_xy = true") + conn.execute("SET GLOBAL TimeZone = 'UTC'") + + # Cache parquet files metadata (footers) in memory, so each query doesn't + # fetch it again (expensive for remote files with many row groups). + conn.execute("SET GLOBAL parquet_metadata_cache = true") + + # Attach the main database file if specified in settings. + if settings.database: + # use the file name (without extension) as alias, also for URLs + alias = pathlib.PurePosixPath(urlparse(settings.database).path).stem + logger.debug(f"Attaching DuckDB database {settings.database} as {alias}") + conn.execute( + f"ATTACH {_quote_literal(settings.database)} AS {_quote_ident(alias)} (READ_ONLY)" + ) + + # Attach and register file sources as views in the in-memory database. + for name, path in settings.sources.items(): + logger.debug(f"Registering DuckDB source {name}: {path}") + view = f"CREATE OR REPLACE VIEW memory.main.{_quote_ident(name)} AS" + + # Single parquet file: add the row position in the file as a stable + # feature id (looking up a row position only reads one row group). + # Not for globs, where row positions are not unique. + if _is_parquet(path) and not _is_glob(path): + try: + conn.execute(f"{view} {_source_query(path, file_row_number=True)}") + continue + except duckdb.BinderException as e: + # e.g the file already has a `file_row_number` column + logger.debug(f"Could not add `file_row_number` to {name}: {e}") + + conn.execute(f"{view} {_source_query(path)}") + + return conn + + +async def connect_to_duckdb( + app: FastAPI, + *, + settings: DuckDBSettings | None = None, +) -> None: + """Connect to DuckDB and store the connection in `app.state.duckdb_connection`.""" + if not settings: + settings = DuckDBSettings() + + app.state.duckdb_connection = create_connection(settings) + + +async def close_duckdb_connection(app: FastAPI) -> None: + """Close the DuckDB connection.""" + app.state.duckdb_connection.close() diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py new file mode 100644 index 00000000..4f8525b1 --- /dev/null +++ b/tipg/duck/settings.py @@ -0,0 +1,65 @@ +"""tipg DuckDB settings.""" + +from pydantic_settings import BaseSettings + + +class DuckDBSettings(BaseSettings): + """DuckDB connection settings. + + Attributes: + database: path or URL to a DuckDB database file, attached (read-only) + under its file name (without extension). If not set, only `sources` + are served. + threads: number of threads DuckDB can use. + memory_limit: maximum memory DuckDB can use (e.g `4GB`). + sources: mapping of collection id to a file path/glob/URL. Parquet files + are read with `read_parquet`, other formats with the spatial + extension's `ST_Read`. Each source is registered as a view in the + in-memory `memory.main` schema. + + """ + + database: str | None = None + threads: int | None = None + memory_limit: str | None = None + sources: dict[str, str] = {} + + model_config = { + "env_prefix": "TIPG_DUCKDB_", + "env_file": ".env", + "extra": "ignore", + } + + +class DuckDBCatalogSettings(BaseSettings): + """TiPg DuckDB catalog settings. + + Attributes: + spatial_extent: read the spatial extent and geometry type of parquet + sources from their GeoParquet metadata (`bbox` and `geometry_types`). + Only the file footers are read, rows are never scanned. For a glob, + the footer of every matched file is read (one request per file for + remote files), on startup and on each catalog refresh: disable it + for sources matching many (remote) files. When disabled, or when + the metadata is missing, collections have no spatial extent + (world bbox) and a generic `Geometry` type. Tables and other views + never have a spatial extent. The GeoParquet bbox `covering` + columns (if any) are also read from this metadata and used to + speed up spatial filters (`bbox`, tiles, cql2 spatial predicates). + metadata_threads: number of parquet sources whose metadata is read + concurrently when building the catalog. + + """ + + schemas: list[str] = ["main"] + tables: list[str] | None = None + exclude_tables: list[str] | None = None + spatial_extent: bool = True + metadata_threads: int = 16 + only_spatial_tables: bool = True + + model_config = { + "env_prefix": "TIPG_DUCKDB_", + "env_file": ".env", + "extra": "ignore", + } diff --git a/tipg/errors.py b/tipg/errors.py index a6ef9e1c..6b4fb200 100644 --- a/tipg/errors.py +++ b/tipg/errors.py @@ -68,6 +68,10 @@ class MissingCollectionCatalog(TiPgError): """`collection_catalog` not registered in the application state.""" +class TipgExperimentalWarning(UserWarning): + """Warning for experimental features in TiPg.""" + + DEFAULT_STATUS_CODES = { NotFound: status.HTTP_404_NOT_FOUND, InvalidBBox: status.HTTP_422_UNPROCESSABLE_CONTENT, diff --git a/tipg/pg/collections.py b/tipg/pg/collections.py index 3004183c..6baefc38 100644 --- a/tipg/pg/collections.py +++ b/tipg/pg/collections.py @@ -9,7 +9,7 @@ from morecantile import Tile, TileMatrixSet from pydantic import ConfigDict, Field -from ..collections import Collection, Column, Feature, ItemList, _quote_ident +from ..collections import Collection, Column, Feature, ItemList, Srid, _quote_ident from ..errors import ( InvalidGeometryColumnName, InvalidLimit, @@ -63,11 +63,11 @@ def _qualified_name(self) -> str: """SQL-quoted ``"schema"."table"`` identifier for this collection.""" return f"{_quote_ident(self.dbschema)}.{_quote_ident(self.table)}" - def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Tag a geometry literal with its SRID (PostGIS ``ST_SetSRID``).""" return {"op": "st_setsrid", "args": [geom, srid]} - def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Reproject a EPSG:4326 geometry literal to ``srid`` (PostGIS ``ST_Transform``).""" return {"op": "st_transform", "args": [geom, srid]} diff --git a/uv.lock b/uv.lock index ebf95497..8828dc86 100644 --- a/uv.lock +++ b/uv.lock @@ -3,8 +3,8 @@ revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12'", "python_full_version < '3.11.5'", @@ -1009,6 +1009,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl", hash = "sha256:cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638", size = 472099, upload-time = "2026-10-16T13:56:32.053Z" }, ] +[[package]] +name = "duckdb" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/7c/dfdf5e9cebefd032f57f09b224993e861ecb55f61e36c709544b04b44f70/duckdb-2.0.0.tar.gz", hash = "sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f", size = 16771077, upload-time = "2026-10-21T12:51:13.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/57/ec7c1a01ab43fc13c55c17cc54a5740886e04ae830aa420ac00614b1420a/duckdb-2.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a3b38d8cdbef10007c64b65efc4087f799c4dc606785aa5f2a425ed13658ca01", size = 37006608, upload-time = "2026-10-21T12:50:38.611Z" }, + { url = "https://files.pythonhosted.org/packages/84/36/e1ea89acd74917ca654d5adc278765f0116e07540abf4134d5cf9f527363/duckdb-2.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:9bcc9921be0e1b2404b96a7fb447a8b80fe2f3a919cdc976ca6c39951cc8f2f5", size = 19691094, upload-time = "2026-10-21T12:50:41.839Z" }, + { url = "https://files.pythonhosted.org/packages/4b/81/2895b9ee3858dae7a57f791d7dbd4f116e9fa8fd3c71b9a377dbd4d292a7/duckdb-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61a244faae9fd787e30f4c76888769ec49e79736c434f698e8178a00af8348f0", size = 17603089, upload-time = "2026-10-21T12:50:44.167Z" }, + { url = "https://files.pythonhosted.org/packages/04/95/0c8b5b95ac9e14ee83090a78048fbb23fb9f8abacd63e11a5eee7fcb31e1/duckdb-2.0.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcdf10dfbc2c1a6bdc5b2a94bba5b15481e552c5745c2af1994617b0eb6bc750", size = 21637946, upload-time = "2026-10-21T12:50:46.444Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a1/9c3780faada6e6a3edc5f9364cf19810e6e1ea90a7d5fd8fffc48b42c269/duckdb-2.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a3601121c7cc50d0b70412995634682fafcb58a0156d5d663dfd0b3bbe30a33", size = 24351483, upload-time = "2026-10-21T12:50:48.902Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/0729ac5d9857e5f0bde7262961f80e578a29945a444169886094cc9c69f4/duckdb-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a387d4907b535a4bd82ba8aede6809d0bf023dfbb21362832f3c9848aea7743c", size = 15370691, upload-time = "2026-10-21T12:50:51.59Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/392991ec9dcb22811c7029009184a910bd5cb69fbb7a626f36f151215535/duckdb-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:62bea9a1786be98d2906197c67a4089cc8b34d66d19e87e5543d8201e81323ad", size = 15731407, upload-time = "2026-10-21T12:50:53.889Z" }, + { url = "https://files.pythonhosted.org/packages/41/28/1fb52f9ea10205a55a07c32e37cd4fd68fa9d1dc2520a12405ec9bb59cac/duckdb-2.0.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9", size = 37003294, upload-time = "2026-10-21T12:50:56.489Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/fbe92644709fb9162cc51753f92f892b1f333658079d1f11efaed43341d2/duckdb-2.0.0-cp312-abi3-macosx_10_15_x86_64.whl", hash = "sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd", size = 19689207, upload-time = "2026-10-21T12:50:59.331Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/4d23938223626980cdcf18fd6b0dbf040823b5cf15beabbf4ab7ee276631/duckdb-2.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef", size = 17597612, upload-time = "2026-10-21T12:51:01.818Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/985fb9d93d7f9f8e82092ddfe05ec8c9a1d0df7bdb0f562a10ced30e3e8f/duckdb-2.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa", size = 21655920, upload-time = "2026-10-21T12:51:04.36Z" }, + { url = "https://files.pythonhosted.org/packages/cc/33/01ad2535ee897db2bad70cbc800655167c53afd0b955eed84f0759c2f417/duckdb-2.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431", size = 24371005, upload-time = "2026-10-21T12:51:06.86Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/9e8ca1242b98f04041bef09686ecfe098b41f820aa23240f63f360b64eaf/duckdb-2.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a", size = 15368860, upload-time = "2026-10-21T12:51:09.395Z" }, + { url = "https://files.pythonhosted.org/packages/37/81/6ea5dc1050f86ec4dd038a2774766b42247ba6104e7785f13a0c600e2b33/duckdb-2.0.0-cp312-abi3-win_arm64.whl", hash = "sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873", size = 15726489, upload-time = "2026-10-21T12:51:11.643Z" }, +] + [[package]] name = "executing" version = "2.3.0" @@ -2279,8 +2301,8 @@ version = "2.5.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", ] sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } @@ -3183,6 +3205,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] +[[package]] +name = "pytz" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/b0/168030b0bfcc334cf09fa3313afdb888d892c8a0027324683d1de0ec277e/pytz-2026.6.tar.gz", hash = "sha256:3b0fc2727707663d6e517936f444fd8a8b240f1fc32f4bf7ee03274245945284", size = 318520, upload-time = "2026-10-29T09:05:47.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/25/564f3fc5b70c1398d706c51f2b1470d0ea0b1b2f96fee67877b6f5bdf3b6/pytz-2026.6-py2.py3-none-any.whl", hash = "sha256:7fed3fee8fd6bd4a2b7b75a8d56021d0624127bf69bd5691f115db929ac570b3", size = 506321, upload-time = "2026-10-29T09:05:45.14Z" }, +] + [[package]] name = "pywinpty" version = "3.1.2" @@ -3689,8 +3720,8 @@ version = "4.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12'", ] @@ -3784,6 +3815,10 @@ dependencies = [ ] [package.optional-dependencies] +duckdb = [ + { name = "duckdb" }, + { name = "pytz" }, +] metrics = [ { name = "prometheus-fastapi-instrumentator" }, ] @@ -3804,7 +3839,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, - { name = "tipg", extra = ["metrics"] }, + { name = "tipg", extra = ["duckdb", "metrics"] }, ] docs = [ { name = "black" }, @@ -3824,6 +3859,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.23.0" }, { name = "ciso8601", specifier = "~=2.3" }, { name = "cql2", specifier = ">=0.5,<0.7" }, + { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=2.0" }, { name = "fastapi", specifier = ">=0.100.0" }, { name = "geojson-pydantic", specifier = ">=1.0,<3.0" }, { name = "jinja2", specifier = ">=2.11.2,<4.0.0" }, @@ -3832,10 +3868,11 @@ requires-dist = [ { name = "prometheus-fastapi-instrumentator", marker = "extra == 'metrics'", specifier = ">=8.0.2" }, { name = "pydantic", specifier = ">=2.4,<3.0" }, { name = "pydantic-settings", specifier = "~=2.0" }, + { name = "pytz", marker = "extra == 'duckdb'" }, { name = "starlette", specifier = ">=1.0" }, { name = "starlette-cramjam", specifier = ">=0.4,<1.0" }, ] -provides-extras = ["metrics"] +provides-extras = ["duckdb", "metrics"] [package.metadata.requires-dev] dev = [ @@ -3852,6 +3889,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, + { name = "tipg", extras = ["duckdb"] }, { name = "tipg", extras = ["metrics"] }, ] docs = [