From 487efecec58c250df642243e58d617ed8186408b Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 15:20:35 +0100 Subject: [PATCH 01/22] build: add duckdb optional dependency Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 6 ++++++ uv.lock | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8d86796..92b59ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "folium", "bump-my-version", "tipg[metrics]", + "tipg[duckdb]", ] server = [ "uvicorn[standard]>=0.12.0", @@ -69,6 +70,11 @@ docs = [ metrics = [ "prometheus-fastapi-instrumentator>=8.0.2", ] +duckdb = [ + "duckdb>=1.4", + # required by duckdb to return TIMESTAMP WITH TIME ZONE values to Python + "pytz", +] [project.urls] Homepage = "https://developmentseed.org/tipg" diff --git a/uv.lock b/uv.lock index ebf95497..8e087022 100644 --- a/uv.lock +++ b/uv.lock @@ -1009,6 +1009,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl", hash = "sha256:cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638", size = 472099, upload-time = "2026-10-16T13:56:32.053Z" }, ] +[[package]] +name = "duckdb" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/7c/dfdf5e9cebefd032f57f09b224993e861ecb55f61e36c709544b04b44f70/duckdb-2.0.0.tar.gz", hash = "sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f", size = 16771077, upload-time = "2026-10-21T12:51:13.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/57/ec7c1a01ab43fc13c55c17cc54a5740886e04ae830aa420ac00614b1420a/duckdb-2.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a3b38d8cdbef10007c64b65efc4087f799c4dc606785aa5f2a425ed13658ca01", size = 37006608, upload-time = "2026-10-21T12:50:38.611Z" }, + { url = "https://files.pythonhosted.org/packages/84/36/e1ea89acd74917ca654d5adc278765f0116e07540abf4134d5cf9f527363/duckdb-2.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:9bcc9921be0e1b2404b96a7fb447a8b80fe2f3a919cdc976ca6c39951cc8f2f5", size = 19691094, upload-time = "2026-10-21T12:50:41.839Z" }, + { url = "https://files.pythonhosted.org/packages/4b/81/2895b9ee3858dae7a57f791d7dbd4f116e9fa8fd3c71b9a377dbd4d292a7/duckdb-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61a244faae9fd787e30f4c76888769ec49e79736c434f698e8178a00af8348f0", size = 17603089, upload-time = "2026-10-21T12:50:44.167Z" }, + { url = "https://files.pythonhosted.org/packages/04/95/0c8b5b95ac9e14ee83090a78048fbb23fb9f8abacd63e11a5eee7fcb31e1/duckdb-2.0.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcdf10dfbc2c1a6bdc5b2a94bba5b15481e552c5745c2af1994617b0eb6bc750", size = 21637946, upload-time = "2026-10-21T12:50:46.444Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a1/9c3780faada6e6a3edc5f9364cf19810e6e1ea90a7d5fd8fffc48b42c269/duckdb-2.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a3601121c7cc50d0b70412995634682fafcb58a0156d5d663dfd0b3bbe30a33", size = 24351483, upload-time = "2026-10-21T12:50:48.902Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/0729ac5d9857e5f0bde7262961f80e578a29945a444169886094cc9c69f4/duckdb-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a387d4907b535a4bd82ba8aede6809d0bf023dfbb21362832f3c9848aea7743c", size = 15370691, upload-time = "2026-10-21T12:50:51.59Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/392991ec9dcb22811c7029009184a910bd5cb69fbb7a626f36f151215535/duckdb-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:62bea9a1786be98d2906197c67a4089cc8b34d66d19e87e5543d8201e81323ad", size = 15731407, upload-time = "2026-10-21T12:50:53.889Z" }, + { url = "https://files.pythonhosted.org/packages/41/28/1fb52f9ea10205a55a07c32e37cd4fd68fa9d1dc2520a12405ec9bb59cac/duckdb-2.0.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9", size = 37003294, upload-time = "2026-10-21T12:50:56.489Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/fbe92644709fb9162cc51753f92f892b1f333658079d1f11efaed43341d2/duckdb-2.0.0-cp312-abi3-macosx_10_15_x86_64.whl", hash = "sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd", size = 19689207, upload-time = "2026-10-21T12:50:59.331Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/4d23938223626980cdcf18fd6b0dbf040823b5cf15beabbf4ab7ee276631/duckdb-2.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef", size = 17597612, upload-time = "2026-10-21T12:51:01.818Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/985fb9d93d7f9f8e82092ddfe05ec8c9a1d0df7bdb0f562a10ced30e3e8f/duckdb-2.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa", size = 21655920, upload-time = "2026-10-21T12:51:04.36Z" }, + { url = "https://files.pythonhosted.org/packages/cc/33/01ad2535ee897db2bad70cbc800655167c53afd0b955eed84f0759c2f417/duckdb-2.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431", size = 24371005, upload-time = "2026-10-21T12:51:06.86Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/9e8ca1242b98f04041bef09686ecfe098b41f820aa23240f63f360b64eaf/duckdb-2.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a", size = 15368860, upload-time = "2026-10-21T12:51:09.395Z" }, + { url = "https://files.pythonhosted.org/packages/37/81/6ea5dc1050f86ec4dd038a2774766b42247ba6104e7785f13a0c600e2b33/duckdb-2.0.0-cp312-abi3-win_arm64.whl", hash = "sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873", size = 15726489, upload-time = "2026-10-21T12:51:11.643Z" }, +] + [[package]] name = "executing" version = "2.3.0" @@ -3183,6 +3205,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] +[[package]] +name = "pytz" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/b0/168030b0bfcc334cf09fa3313afdb888d892c8a0027324683d1de0ec277e/pytz-2026.6.tar.gz", hash = "sha256:3b0fc2727707663d6e517936f444fd8a8b240f1fc32f4bf7ee03274245945284", size = 318520, upload-time = "2026-10-29T09:05:47.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/25/564f3fc5b70c1398d706c51f2b1470d0ea0b1b2f96fee67877b6f5bdf3b6/pytz-2026.6-py2.py3-none-any.whl", hash = "sha256:7fed3fee8fd6bd4a2b7b75a8d56021d0624127bf69bd5691f115db929ac570b3", size = 506321, upload-time = "2026-10-29T09:05:45.14Z" }, +] + [[package]] name = "pywinpty" version = "3.1.2" @@ -3784,6 +3815,10 @@ dependencies = [ ] [package.optional-dependencies] +duckdb = [ + { name = "duckdb" }, + { name = "pytz" }, +] metrics = [ { name = "prometheus-fastapi-instrumentator" }, ] @@ -3804,7 +3839,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, - { name = "tipg", extra = ["metrics"] }, + { name = "tipg", extra = ["duckdb", "metrics"] }, ] docs = [ { name = "black" }, @@ -3824,6 +3859,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.23.0" }, { name = "ciso8601", specifier = "~=2.3" }, { name = "cql2", specifier = ">=0.5,<0.7" }, + { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.4" }, { name = "fastapi", specifier = ">=0.100.0" }, { name = "geojson-pydantic", specifier = ">=1.0,<3.0" }, { name = "jinja2", specifier = ">=2.11.2,<4.0.0" }, @@ -3832,10 +3868,11 @@ requires-dist = [ { name = "prometheus-fastapi-instrumentator", marker = "extra == 'metrics'", specifier = ">=8.0.2" }, { name = "pydantic", specifier = ">=2.4,<3.0" }, { name = "pydantic-settings", specifier = "~=2.0" }, + { name = "pytz", marker = "extra == 'duckdb'" }, { name = "starlette", specifier = ">=1.0" }, { name = "starlette-cramjam", specifier = ">=0.4,<1.0" }, ] -provides-extras = ["metrics"] +provides-extras = ["duckdb", "metrics"] [package.metadata.requires-dev] dev = [ @@ -3852,6 +3889,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, + { name = "tipg", extras = ["duckdb"] }, { name = "tipg", extras = ["metrics"] }, ] docs = [ From a39c89d4960dced5f7f6e70a89ecdf7655debac7 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:05:07 +0100 Subject: [PATCH 02/22] feat(duck): add DuckDB settings and connection handling Co-Authored-By: Claude Opus 5.5 --- tipg/duck/database.py | 85 +++++++++++++++++++++++++++++++++++++++++++ tipg/duck/settings.py | 49 +++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tipg/duck/database.py create mode 100644 tipg/duck/settings.py diff --git a/tipg/duck/database.py b/tipg/duck/database.py new file mode 100644 index 00000000..bc924b5e --- /dev/null +++ b/tipg/duck/database.py @@ -0,0 +1,85 @@ +"""tipg DuckDB database events.""" + +import pathlib + +import duckdb +from fastapi import FastAPI + +from ..collections import _quote_ident +from ..logger import logger +from .settings import DuckDBSettings + + +def _quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def _source_query(path: str) -> str: + """SQL `SELECT` for a file source.""" + if path.lower().endswith((".parquet", ".parq", ".geoparquet")): + return f"SELECT * FROM read_parquet({_quote_literal(path)})" + + return f"SELECT * FROM ST_Read({_quote_literal(path)})" + + +def create_connection( + settings: DuckDBSettings | None = None, +) -> duckdb.DuckDBPyConnection: + """Create and configure a DuckDB connection. + + The connection always uses an in-memory primary database, where file + `sources` are registered as views. When `settings.database` points to a + database file, it is attached (read-only by default) under its file name + (without extension). + + Note: views (and settings) must be visible from `connection.cursor()`, + which creates a new connection to the same database instance, so we + can't use TEMP views (connection-local) nor session-level `SET`. + """ + settings = settings or DuckDBSettings() + + config: dict[str, str | int] = {} + if settings.threads: + config["threads"] = settings.threads + if settings.memory_limit: + config["memory_limit"] = settings.memory_limit + + conn = duckdb.connect(":memory:", config=config) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + + # DuckDB GEOMETRY uses [EASTING, NORTHING] (lon/lat) order internally, + # make sure ST_Transform and GDAL follow it. + conn.execute("SET GLOBAL geometry_always_xy = true") + conn.execute("SET GLOBAL TimeZone = 'UTC'") + + if settings.database not in ("", ":memory:"): + alias = pathlib.Path(settings.database).stem + options = " (READ_ONLY)" if settings.read_only else "" + logger.debug(f"Attaching DuckDB database {settings.database} as {alias}") + conn.execute( + f"ATTACH {_quote_literal(settings.database)} AS {_quote_ident(alias)}{options}" + ) + + for name, path in settings.sources.items(): + logger.debug(f"Registering DuckDB source {name}: {path}") + conn.execute( + f"CREATE OR REPLACE VIEW memory.main.{_quote_ident(name)} AS {_source_query(path)}" + ) + + return conn + + +async def connect_to_db( + app: FastAPI, + *, + settings: DuckDBSettings | None = None, +) -> None: + """Connect to DuckDB and store the connection in `app.state.duckdb_connection`.""" + app.state.duckdb_connection = create_connection(settings) + + +async def close_db_connection(app: FastAPI) -> None: + """Close the DuckDB connection.""" + app.state.duckdb_connection.close() diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py new file mode 100644 index 00000000..6d84df4f --- /dev/null +++ b/tipg/duck/settings.py @@ -0,0 +1,49 @@ +"""tipg DuckDB settings.""" + +from pydantic_settings import BaseSettings + + +class DuckDBSettings(BaseSettings): + """DuckDB connection settings. + + Attributes: + database: path to a DuckDB database file, attached under its file name + (default to an in-memory database only). + read_only: attach the database file in read-only mode. + threads: number of threads DuckDB can use. + memory_limit: maximum memory DuckDB can use (e.g `4GB`). + sources: mapping of collection id to a file path/glob/URL. Parquet files + are read with `read_parquet`, other formats with the spatial + extension's `ST_Read`. Each source is registered as a view in the + in-memory `memory.main` schema. + + """ + + database: str = ":memory:" + read_only: bool = True + threads: int | None = None + memory_limit: str | None = None + sources: dict[str, str] = {} + + model_config = { + "env_prefix": "TIPG_DUCKDB_", + "env_file": ".env", + "extra": "ignore", + } + + +class DuckDBCatalogSettings(BaseSettings): + """TiPg DuckDB catalog settings.""" + + schemas: list[str] = ["main"] + tables: list[str] | None = None + exclude_tables: list[str] | None = None + datetime_extent: bool = True + spatial_extent: bool = True + only_spatial_tables: bool = True + + model_config = { + "env_prefix": "TIPG_DUCKDB_", + "env_file": ".env", + "extra": "ignore", + } From 946354e239ebfec9f35fe5d98b5b67bd95bade25 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:07:22 +0100 Subject: [PATCH 03/22] feat(duck): add DuckCollection (features and vector tiles) Mirrors PgCollection: same query structure, validation and output. Queries run in a threadpool on a per-call cursor of the collection's DuckDB connection; CQL geometry literals are tagged/reprojected with DuckDB's ST_SetCRS / ST_Transform; MVT properties DuckDB can't encode are cast. Co-Authored-By: Claude Opus 5.5 --- tipg/duck/collections.py | 442 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 tipg/duck/collections.py diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py new file mode 100644 index 00000000..18f5fb27 --- /dev/null +++ b/tipg/duck/collections.py @@ -0,0 +1,442 @@ +"""tipg DuckDB collection models.""" + +import re +from typing import Any + +import duckdb +import orjson +from cql2 import Expr +from morecantile import Tile, TileMatrixSet +from pydantic import ConfigDict, Field +from starlette.concurrency import run_in_threadpool + +from ..collections import Collection, Column, Feature, ItemList, _quote_ident +from ..errors import InvalidGeometryColumnName, InvalidLimit, InvalidPropertyName +from ..logger import logger +from ..settings import FeaturesSettings, MVTSettings + +mvt_settings = MVTSettings() +features_settings = FeaturesSettings() # type: ignore [call-arg] + +# Column types (as normalized in the catalog) ST_AsMVT can encode as-is +_MVT_NATIVE_TYPES = frozenset( + { + "smallint", + "integer", + "bigint", + "real", + "float8", + "boolean", + "text", + "json", + } +) + + +def _quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def _crs(srid: int) -> str: + """DuckDB CRS identifier for an EPSG code.""" + return f"EPSG:{srid}" + + +class DuckCollection(Collection): + """Model for DuckDB Table and View.""" + + database: str + dbschema: str = Field(alias="schema") + + # Reference to the (shared) DuckDB connection serving this collection. + # Excluded from serialization and repr. + connection: duckdb.DuckDBPyConnection = Field(exclude=True, repr=False) + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @property + def _qualified_name(self) -> str: + """SQL-quoted ``"database"."schema"."table"`` identifier for this collection.""" + return ".".join( + _quote_ident(p) for p in (self.database, self.dbschema, self.table) + ) + + def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + """Tag a geometry literal with its CRS (DuckDB ``ST_SetCRS``).""" + return {"op": "st_setcrs", "args": [geom, _crs(srid)]} + + def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + """Reproject a EPSG:4326 geometry literal to ``srid`` (DuckDB ``ST_Transform``).""" + return {"op": "st_transform", "args": [geom, _crs(4326), _crs(srid)]} + + def _select_property_columns(self, properties: list[str] | None) -> list[str]: + """Quoted column-name fragments for the property columns to SELECT.""" + return [_quote_ident(c) for c in self.columns(properties)] + + def _select_mvt_property_columns(self, properties: list[str] | None) -> list[str]: + """Property columns to SELECT for MVT encoding. + + ``ST_AsMVT`` only supports boolean, integer, float and string values: + decimals are cast to DOUBLE and other types to VARCHAR. + """ + cols = [] + for name in self.columns(properties): + col = self.get_column(name) + q = _quote_ident(name) + if col is None or col.type in _MVT_NATIVE_TYPES: + cols.append(q) + elif col.type == "numeric": + cols.append(f"CAST({q} AS DOUBLE) AS {q}") + else: + cols.append(f"CAST({q} AS VARCHAR) AS {q}") + return cols + + def _select_id(self) -> str: + """SQL fragment that selects the primary-key column as ``tipg_id``. + + Falls back to ``ROW_NUMBER()`` when the collection has no primary key. + """ + if self.id_column: + return f"{_quote_ident(self.id_column.name)} AS tipg_id" + return "ROW_NUMBER() OVER () AS tipg_id" + + def _geom_expr( + self, + geometry_column: Column | None, + bbox_only: bool | None, + simplify: float | None, + ) -> str | None: + """Build the SQL expression for the geometry column (reprojected to + 4326 if needed, optionally bbox-only or simplified). + """ + if geometry_column is None: + return None + + g = _quote_ident(geometry_column.name) + + if geometry_column.srid != 4326: + g = f"ST_Transform({g}, '{_crs(geometry_column.srid)}', '{_crs(4326)}')" # type: ignore [arg-type] + + if bbox_only: + g = f"ST_Envelope({g})" + elif simplify: + s = float(simplify) + g = f"ST_ReducePrecision(ST_SimplifyPreserveTopology({g}, {s}), {s})" + + return g + + def _select( + self, + properties: list[str] | None, + geometry_column: Column | None, + bbox_only: bool | None, + simplify: float | None, + geom_as_wkt: bool = False, + ) -> str: + """Build the SELECT clause for the main features query.""" + cols = self._select_property_columns(properties) + cols.append(self._select_id()) + + geom = self._geom_expr(geometry_column, bbox_only, simplify) + if geom_as_wkt: + cols.append( + f"'SRID=4326;' || ST_AsText({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS VARCHAR) AS tipg_geom" + ) + else: + cols.append( + f"ST_AsGeoJSON({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS VARCHAR) AS tipg_geom" + ) + + return "SELECT " + ", ".join(cols) + + def _select_mvt( + self, + properties: list[str] | None, + geometry_column: Column, + tms: TileMatrixSet, + tile: Tile, + simplify: float | None = None, + ) -> str: + """Build the SELECT clause that emits an MVT geometry per row.""" + cols = self._select_mvt_property_columns(properties) + + geom = _quote_ident(geometry_column.name) + src_crs = _crs(geometry_column.srid) # type: ignore [arg-type] + + if simplify: + geom = f"ST_ReducePrecision(ST_SimplifyPreserveTopology({geom}, {simplify}), {simplify})" + + # For tiles outside the TMS's natural domain, clip the geometry to the + # TMS's geographic bbox before reprojecting (see PgCollection._select_mvt). + if not tms.is_valid(tile, strict=False): + west, south, east, north = tms.bbox + geo_crs = _crs(tms.geographic_crs.to_epsg() or 4326) + geom = ( + f"ST_Intersection(" + f"ST_MakeEnvelope({west}, {south}, {east}, {north}), " + f"ST_Transform({geom}, '{src_crs}', '{geo_crs}')" + f")" + ) + src_crs = geo_crs + + # Reproject to TMS CRS — prefer EPSG code, fall back to PROJ string. + if tms_srid := tms.crs.to_epsg(): + tms_crs = _crs(tms_srid) + else: + tms_crs = tms.crs.to_proj4() + + transformed = f"ST_Transform({geom}, {_quote_literal(src_crs)}, {_quote_literal(tms_crs)})" + + bbox = tms.xy_bounds(tile) + envelope = f"ST_Extent(ST_MakeEnvelope({bbox.left}, {bbox.bottom}, {bbox.right}, {bbox.top}))" + cols.append( + f"ST_AsMVTGeom(" + f"{transformed}, {envelope}, " + f"{int(mvt_settings.tile_resolution)}, " + f"{int(mvt_settings.tile_buffer)}, " + f"{'TRUE' if mvt_settings.tile_clip else 'FALSE'}" + f") AS geom" + ) + + return "SELECT " + ", ".join(cols) + + def _from(self) -> str: + """Build the FROM clause.""" + return f"FROM {self._qualified_name}" + + def _sort_expressions(self, sortby: str | None) -> list[tuple[str, str]]: + """Parse ``sortby`` into ``(column name, SQL fragment)`` pairs.""" + if sortby: + sorts = [] + for s in sortby.strip().split(","): + parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore + + direction = parts["direction"] + column = parts["column"].strip() + if not self.get_column(column): + raise InvalidPropertyName(f"Property {column} does not exist.") + col_sql = _quote_ident(column) + sorts.append( + (column, f"{col_sql} DESC" if direction == "-" else col_sql) + ) + return sorts + + if self.id_column is not None: + column = self.id_column.name + else: + column = self.properties[0].name + + return [(column, _quote_ident(column))] + + def _sortby(self, sortby: str | None) -> str: + """Build the ORDER BY clause.""" + return "ORDER BY " + ", ".join(sql for _, sql in self._sort_expressions(sortby)) + + @staticmethod + def _where_clause(where: Expr | None) -> str: + """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" + if where is None: + return "WHERE TRUE" + return f"WHERE {where.to_sql()}" + + def _row_to_feature(self, row: dict[str, Any], geom_as_wkt: bool) -> Feature: + """Convert a result row to a Feature.""" + g = row.pop("tipg_geom") + id = row.pop("tipg_id") + if g is not None and not geom_as_wkt: + g = orjson.loads(g) + + # JSON columns are returned as strings + for name, value in row.items(): + if isinstance(value, str) and (col := self.get_column(name)): + if col.type == "json": + row[name] = orjson.loads(value) + + return Feature(type="Feature", geometry=g, id=id, properties=row) + + def _run_features_queries( + self, count_sql: str, sql: str, geom_as_wkt: bool + ) -> tuple[int, list[Feature]]: + """Run the COUNT and Features queries (blocking).""" + cursor = self.connection.cursor() + try: + logger.debug(count_sql) + matched = cursor.execute(count_sql).fetchone()[0] # type: ignore [index] + + logger.debug(sql) + cursor.execute(sql) + names = [d[0] for d in cursor.description] # type: ignore [union-attr] + features = [ + self._row_to_feature(dict(zip(names, r)), geom_as_wkt) + for r in cursor.fetchall() + ] + finally: + cursor.close() + + return matched, features + + def _run_tile_query(self, sql: str, layer: str) -> bytes: + """Run the MVT query (blocking).""" + cursor = self.connection.cursor() + try: + logger.debug(sql) + tile = cursor.execute(sql, [layer]).fetchone()[0] # type: ignore [index] + finally: + cursor.close() + + return bytes(tile) if tile is not None else b"" + + async def features( + self, + *, + ids_filter: list[str] | None = None, + bbox_filter: list[float] | None = None, + datetime_filter: list[str] | None = None, + properties_filter: list[tuple[str, str]] | None = None, + cql_filter: Expr | None = None, + sortby: str | None = None, + properties: list[str] | None = None, + geom: str | None = None, + dt: str | None = None, + limit: int | None = None, + offset: int | None = None, + bbox_only: bool | None = None, + simplify: float | None = None, + geom_as_wkt: bool = False, + function_parameters: dict[str, str] | None = None, + ) -> ItemList: + """Build and run DuckDB query.""" + limit = limit or features_settings.default_features_limit + offset = offset or 0 + + if geom and geom.lower() != "none" and not self.get_geometry_column(geom): + raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") + + if limit and limit > features_settings.max_features_per_query: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" + ) + + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + ) + where = self._where_clause(where_filter) + + count_sql = " ".join(["SELECT COUNT(*)", self._from(), where]) + sql = " ".join( + [ + self._select( + properties=properties, + geometry_column=self.get_geometry_column(geom), + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + ), + self._from(), + where, + self._sortby(sortby), + f"LIMIT {int(limit)}", + f"OFFSET {int(offset)}", + ] + ) + + matched, features = await run_in_threadpool( + self._run_features_queries, count_sql, sql, geom_as_wkt + ) + returned = len(features) + + return ItemList( + items=features, + matched=matched, + next=offset + returned if matched - returned > offset else None, + prev=max(offset - limit, 0) if offset else None, + ) + + async def get_tile( + self, + *, + tms: TileMatrixSet, + tile: Tile, + ids_filter: list[str] | None = None, + bbox_filter: list[float] | None = None, + datetime_filter: list[str] | None = None, + properties_filter: list[tuple[str, str]] | None = None, + function_parameters: dict[str, str] | None = None, + cql_filter: Expr | None = None, + sortby: str | None = None, + properties: list[str] | None = None, + geom: str | None = None, + dt: str | None = None, + limit: int | None = None, + simplify: float | None = None, + ) -> bytes: + """Build query to get Vector Tile.""" + limit = limit or mvt_settings.max_features_per_tile + + geometry_column = self.get_geometry_column(geom) + if not geometry_column: + raise InvalidGeometryColumnName(f"Invalid Geometry Column Name {geom}") + + if limit > mvt_settings.max_features_per_tile: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_tile` setting of {mvt_settings.max_features_per_tile}" + ) + + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + tms=tms, + tile=tile, + ) + + sorts = self._sort_expressions(sortby) + order_by = "ORDER BY " + ", ".join(sql for _, sql in sorts) + + inner_sql = " ".join( + [ + self._select_mvt( + properties=properties, + geometry_column=geometry_column, + tms=tms, + tile=tile, + simplify=simplify, + ), + self._from(), + self._where_clause(where_filter), + order_by, + f"LIMIT {int(limit)}", + ] + ) + + # Only an ORDER BY inside the aggregate guarantees the feature order + # in the tile. It can only reference columns of ``t``, so an explicit + # sort column must be projected. + aggregate_order = "" + if sortby: + selected = self.columns(properties) + if missing := [col for col, _ in sorts if col not in selected]: + raise InvalidPropertyName( + f"Cannot sort by {', '.join(missing)}: not in the selected `properties`." + ) + aggregate_order = f" {order_by}" + + layer = self.table if mvt_settings.set_mvt_layername is True else "default" + sql = f"WITH t AS ({inner_sql}) SELECT ST_AsMVT(t, $1{aggregate_order}) FROM t" + + return await run_in_threadpool(self._run_tile_query, sql, layer) From 279ffdc78b1e367dc32951653a007de8a1fc8c75 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:08:59 +0100 Subject: [PATCH 04/22] feat(duck): add DuckDB collection catalog duck_get_collection_index lists the tables and views of all attached databases (incl. file sources), normalizes DuckDB types to the names tipg.collections.Column understands, reads the SRID from the geometry column type and computes spatial/temporal extents in one scan per table. Co-Authored-By: Claude Opus 5.5 --- tipg/duck/collections.py | 294 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 1 deletion(-) diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 18f5fb27..20b20995 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -1,5 +1,6 @@ """tipg DuckDB collection models.""" +import datetime import re from typing import Any @@ -13,7 +14,8 @@ from ..collections import Collection, Column, Feature, ItemList, _quote_ident from ..errors import InvalidGeometryColumnName, InvalidLimit, InvalidPropertyName from ..logger import logger -from ..settings import FeaturesSettings, MVTSettings +from ..settings import FeaturesSettings, MVTSettings, TableConfig, TableSettings +from .settings import DuckDBCatalogSettings mvt_settings = MVTSettings() features_settings = FeaturesSettings() # type: ignore [call-arg] @@ -440,3 +442,293 @@ async def get_tile( sql = f"WITH t AS ({inner_sql}) SELECT ST_AsMVT(t, $1{aggregate_order}) FROM t" return await run_in_threadpool(self._run_tile_query, sql, layer) + + +_DUCK_TYPES = { + "BOOLEAN": "boolean", + "TINYINT": "smallint", + "SMALLINT": "smallint", + "UTINYINT": "smallint", + "INTEGER": "integer", + "USMALLINT": "integer", + "BIGINT": "bigint", + "UINTEGER": "bigint", + "UBIGINT": "numeric", + "HUGEINT": "numeric", + "UHUGEINT": "numeric", + "FLOAT": "real", + "DOUBLE": "float8", + "DATE": "date", + "TIMESTAMP": "timestamp", + "TIMESTAMP_S": "timestamp", + "TIMESTAMP_MS": "timestamp", + "TIMESTAMP_NS": "timestamp", + "TIMESTAMP WITH TIME ZONE": "timestamptz", + "JSON": "json", +} + +# ST_GeometryType values → PostGIS-style geometry type names +_GEOMETRY_TYPES = { + "POINT": "Point", + "LINESTRING": "LineString", + "POLYGON": "Polygon", + "MULTIPOINT": "MultiPoint", + "MULTILINESTRING": "MultiLineString", + "MULTIPOLYGON": "MultiPolygon", + "GEOMETRYCOLLECTION": "GeometryCollection", +} + +_GEOMETRY_CRS = re.compile(r"^GEOMETRY\('(?P[^']+)'\)$", re.IGNORECASE) + + +def _normalize_type(data_type: str) -> str: + """Map a DuckDB type name to the type names `tipg.collections.Column` understands.""" + t = data_type.upper() + if t == "GEOMETRY" or _GEOMETRY_CRS.match(t): + return "geometry" + + if t.endswith("[]"): + return _normalize_type(data_type[:-2]) + "[]" + + if t.startswith("DECIMAL"): + return "numeric" + + return _DUCK_TYPES.get(t, "text") + + +def _geometry_srid(data_type: str) -> int: + """SRID of a geometry column, from its type (e.g `GEOMETRY('EPSG:26915')`). + + Geometries without CRS (or in OGC:CRS84) are assumed to be in EPSG:4326. + """ + if m := _GEOMETRY_CRS.match(data_type): + auth, _, code = m.group("crs").partition(":") + if auth.upper() == "EPSG" and code.isdigit(): + return int(code) + + if m.group("crs").upper() not in ("OGC:CRS84", "CRS84"): + logger.warning( + f"Unsupported geometry CRS {m.group('crs')}, assuming EPSG:4326" + ) + + return 4326 + + +def _iso_datetime(value: Any) -> str | None: + """ISO format a date/datetime extent (naive values are assumed to be UTC).""" + if value is None: + return None + + if not isinstance(value, datetime.datetime): + value = datetime.datetime.combine(value, datetime.time()) + + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + + return value.isoformat() + + +def _table_metadata( # noqa: C901 + cursor: duckdb.DuckDBPyConnection, + qualified_name: str, + columns: list[dict[str, Any]], + settings: DuckDBCatalogSettings, +) -> None: + """Add geometry type, bounds and datetime extents to the columns (in place).""" + exprs: list[str] = [] + for c in columns: + q = _quote_ident(c["name"]) + if c["type"] == "geometry": + c["geometry_type"] = "Geometry" + if settings.spatial_extent: + extent = f"ST_Extent_Agg({q})" + if c["srid"] != 4326: + extent = ( + f"ST_Transform({extent}, '{_crs(c['srid'])}', '{_crs(4326)}')" + ) + exprs.extend( + [ + f"ST_XMin({extent})", + f"ST_YMin({extent})", + f"ST_XMax({extent})", + f"ST_YMax({extent})", + f"CAST(min(ST_GeometryType({q})) AS VARCHAR)", + f"CAST(max(ST_GeometryType({q})) AS VARCHAR)", + ] + ) + + elif ( + c["type"] in ("timestamp", "timestamptz", "date") + and settings.datetime_extent + ): + exprs.extend([f"min({q})", f"max({q})"]) + + if not exprs: + return + + values = iter( + cursor.execute(f"SELECT {', '.join(exprs)} FROM {qualified_name}").fetchone() # type: ignore [arg-type] + ) + for c in columns: + if c["type"] == "geometry" and settings.spatial_extent: + xmin, ymin, xmax, ymax, gmin, gmax = (next(values) for _ in range(6)) + if xmin is not None: + c["bounds"] = [xmin, ymin, xmax, ymax] + if gmin is not None and gmin == gmax: + c["geometry_type"] = _GEOMETRY_TYPES.get(gmin.upper(), "Geometry") + + elif ( + c["type"] in ("timestamp", "timestamptz", "date") + and settings.datetime_extent + ): + c["mindt"] = _iso_datetime(next(values)) + c["maxdt"] = _iso_datetime(next(values)) + + +def _collection_index( # noqa: C901 + connection: duckdb.DuckDBPyConnection, + settings: DuckDBCatalogSettings, +) -> list[Collection]: + """Fetch Tables and Views index (blocking).""" + cursor = connection.cursor() + try: + relations = cursor.execute( + """ + SELECT database_name, schema_name, table_name, 'Table', comment + FROM duckdb_tables() + WHERE NOT internal AND NOT temporary + UNION ALL + SELECT database_name, schema_name, view_name, 'View', comment + FROM duckdb_views() + WHERE NOT internal AND NOT temporary + ORDER BY 1, 2, 3 + """ + ).fetchall() + + columns_rows = cursor.execute( + """ + SELECT database_name, schema_name, table_name, column_name, data_type, comment + FROM duckdb_columns() + WHERE NOT internal + ORDER BY database_name, schema_name, table_name, column_index + """ + ).fetchall() + + pk_rows = cursor.execute( + """ + SELECT database_name, schema_name, table_name, constraint_column_names + FROM duckdb_constraints() + WHERE constraint_type = 'PRIMARY KEY' + """ + ).fetchall() + + table_columns: dict[tuple[str, str, str], list[dict[str, Any]]] = {} + for db, schema, table, name, data_type, comment in columns_rows: + col: dict[str, Any] = { + "name": name, + "type": _normalize_type(data_type), + "description": comment, + } + if col["type"] == "geometry": + col["srid"] = _geometry_srid(data_type) + table_columns.setdefault((db, schema, table), []).append(col) + + pks = { + (db, schema, table): names[0] + for db, schema, table, names in pk_rows + if len(names) == 1 + } + + table_settings = TableSettings() + table_confs = table_settings.table_config + fallback_key_names = table_settings.fallback_key_names + + collections: dict[str, Collection] = {} + for db, schema, table, entity, comment in relations: + if schema not in settings.schemas: + continue + + table_id = f"{schema}.{table}" + if settings.tables is not None and table_id not in settings.tables: + continue + + if settings.exclude_tables and table_id in settings.exclude_tables: + continue + + if table_id in collections: + logger.warning( + f"Collection {table_id} already registered, skipping `{db}`.{table_id}" + ) + continue + + columns = table_columns.get((db, schema, table), []) + if settings.only_spatial_tables and not any( + c["type"] == "geometry" for c in columns + ): + continue + + qualified_name = ".".join(_quote_ident(p) for p in (db, schema, table)) + _table_metadata(cursor, qualified_name, columns, settings) + + table_conf = table_confs.get(f"{schema}_{table}", TableConfig()) + + if table_settings.sort_columns: + columns = sorted(columns, key=lambda d: d["name"]) + + cols = [Column(**c) for c in columns] + properties_setting = table_conf.properties or [c.name for c in cols] + + # ID Column + id_column = None + if id_name := table_conf.pk or pks.get((db, schema, table)): + for column in cols: + if id_name == column.name: + id_column = column + break + + if id_column is None and fallback_key_names: + for column in cols: + if column.name in fallback_key_names: + id_column = column + break + + datetime_column = None + geometry_column = None + for column in cols: + if column.is_datetime: + if table_conf.datetimecol == column.name or datetime_column is None: + datetime_column = column + + if column.is_geometry: + if table_conf.geomcol == column.name or geometry_column is None: + geometry_column = column + + collections[table_id] = DuckCollection( + type=entity, + id=table_id, + table=table, + database=db, + schema=schema, + description=comment, + table_columns=cols, + properties=[c for c in cols if c.name in properties_setting], + id_column=id_column, + datetime_column=datetime_column, + geometry_column=geometry_column, + connection=connection, + ) + + finally: + cursor.close() + + return list(collections.values()) + + +async def duck_get_collection_index( + connection: duckdb.DuckDBPyConnection, + settings: DuckDBCatalogSettings | None = None, +) -> list[Collection]: + """Fetch Tables and Views index.""" + return await run_in_threadpool( + _collection_index, connection, settings or DuckDBCatalogSettings() + ) From 29f640d73c618f40149546709f4c279baf45cd55 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:09:00 +0100 Subject: [PATCH 05/22] feat(duck): export public tipg.duck API Co-Authored-By: Claude Opus 5.5 --- tipg/duck/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tipg/duck/__init__.py diff --git a/tipg/duck/__init__.py b/tipg/duck/__init__.py new file mode 100644 index 00000000..08bea969 --- /dev/null +++ b/tipg/duck/__init__.py @@ -0,0 +1,15 @@ +"""tipg.duck package: DuckDB integration for TiPg.""" + +from .collections import DuckCollection, duck_get_collection_index +from .database import close_db_connection, connect_to_db, create_connection +from .settings import DuckDBCatalogSettings, DuckDBSettings + +__all__ = [ + "DuckCollection", + "duck_get_collection_index", + "close_db_connection", + "connect_to_db", + "create_connection", + "DuckDBCatalogSettings", + "DuckDBSettings", +] From cc8c55f6ac81168c1c59517f136330df6f308c0e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:13:31 +0100 Subject: [PATCH 06/22] test(duck): add DuckDB backend tests Cover catalog/extents, items filters (ids, bbox, datetime, properties, cql2 incl. projected CRS), sortby/properties, output formats, item, vector tiles and an app mixing PostgreSQL and DuckDB collections. Co-Authored-By: Claude Opus 5.5 --- tests/duck/__init__.py | 1 + tests/duck/conftest.py | 188 ++++++++++++++++++++++++++ tests/duck/test_duck.py | 289 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 478 insertions(+) create mode 100644 tests/duck/__init__.py create mode 100644 tests/duck/conftest.py create mode 100644 tests/duck/test_duck.py diff --git a/tests/duck/__init__.py b/tests/duck/__init__.py new file mode 100644 index 00000000..31e137eb --- /dev/null +++ b/tests/duck/__init__.py @@ -0,0 +1 @@ +"""DuckDB backend tests.""" diff --git a/tests/duck/conftest.py b/tests/duck/conftest.py new file mode 100644 index 00000000..074b2f15 --- /dev/null +++ b/tests/duck/conftest.py @@ -0,0 +1,188 @@ +"""DuckDB backend test fixtures.""" + +import datetime +from contextlib import asynccontextmanager + +import pytest +from fastapi import FastAPI +from starlette.requests import Request +from starlette.testclient import TestClient + +duckdb = pytest.importorskip("duckdb") + +from tipg.collections import Catalog # noqa: E402 +from tipg.duck import ( # noqa: E402 + DuckDBCatalogSettings, + DuckDBSettings, + close_db_connection, + connect_to_db, + duck_get_collection_index, +) + +FIXTURE_SQL = """ +-- 20 points on a diagonal, integer primary key, datetimes and various property types +CREATE TABLE points ( + id INTEGER PRIMARY KEY, + name VARCHAR, + value DOUBLE, + price DECIMAL(10, 2), + dt TIMESTAMPTZ, + day DATE, + tags VARCHAR[], + props JSON, + geom GEOMETRY +); +INSERT INTO points +SELECT + x::INTEGER, + 'point ' || x, + x * 1.5, + x * 2.25, + TIMESTAMPTZ '2020-01-01 00:00:00+00' + to_hours(x::INTEGER), + DATE '2020-01-01' + x::INTEGER, + ['a', 'b'], + '{"rank": ' || x || '}', + ST_Point(x, x) +FROM range(1, 21) r(x); + +-- polygons, no primary key (`gid` fallback key) +CREATE TABLE polygons AS +SELECT x::INTEGER AS gid, 'polygon ' || x AS name, ST_Buffer(ST_Point(x * 10, x * 10), 1) AS geom +FROM range(1, 4) r(x); + +-- geometries in a projected CRS (UTM zone 15N, around Minneapolis) +CREATE TABLE utm (id INTEGER PRIMARY KEY, name VARCHAR, geom GEOMETRY('EPSG:26915')); +INSERT INTO utm VALUES + (1, 'minneapolis', ST_SetCRS(ST_Point(478000, 4980000), 'EPSG:26915')), + (2, 'duluth', ST_SetCRS(ST_Point(568000, 5182000), 'EPSG:26915')); + +-- no geometry +CREATE TABLE nongeo AS SELECT x::INTEGER AS id, 'row ' || x AS name FROM range(1, 4) r(x); +""" + + +@pytest.fixture(scope="session") +def duckdb_files(tmp_path_factory): + """Create a DuckDB database file and a GeoParquet file.""" + tmp = tmp_path_factory.mktemp("duckdb") + database = str(tmp / "test.duckdb") + parquet = str(tmp / "landmarks.parquet") + + conn = duckdb.connect(database) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + conn.execute(FIXTURE_SQL) + conn.execute( + f"COPY (SELECT id, name, geom FROM points WHERE id <= 5) TO '{parquet}' (FORMAT parquet)" + ) + conn.close() + + return {"database": database, "parquet": parquet} + + +@pytest.fixture +def duckdb_settings(duckdb_files): + """DuckDB settings for the test database.""" + return DuckDBSettings( + database=duckdb_files["database"], + sources={"landmarks": duckdb_files["parquet"]}, + ) + + +def create_duck_app( + settings: DuckDBSettings, + catalog_settings: DuckDBCatalogSettings, +) -> FastAPI: + """Create a TiPg application serving DuckDB collections.""" + # NOTE: imported here (not at module level) so module-level settings are + # read after the environment set by the PostgreSQL test fixtures. + from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers + from tipg.factory import Endpoints + + async def register_collection_catalog(app: FastAPI) -> None: + collections = await duck_get_collection_index( + app.state.duckdb_connection, catalog_settings + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in collections}, + last_updated=datetime.datetime.now(), + ) + + @asynccontextmanager + async def lifespan(app: FastAPI): + await connect_to_db(app, settings=settings) + await register_collection_catalog(app) + yield + await close_db_connection(app) + + app = FastAPI(lifespan=lifespan) + app.include_router(Endpoints(title="TiPg DuckDB").router) + add_exception_handlers(app, DEFAULT_STATUS_CODES) + + @app.get("/rawcatalog") + async def raw_catalog(request: Request): + """Return parsed catalog data for testing.""" + return request.app.state.collection_catalog + + return app + + +@pytest.fixture +def duck_app(duckdb_settings): + """DuckDB-only TiPg app (all tables, spatial or not).""" + app = create_duck_app( + duckdb_settings, DuckDBCatalogSettings(only_spatial_tables=False) + ) + with TestClient(app) as client: + yield client + + +@pytest.fixture +def mixed_app(database_url, duckdb_settings): + """TiPg app serving both PostgreSQL and DuckDB collections.""" + from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers + from tipg.factory import Endpoints + from tipg.pg import ( + DatabaseSettings, + PostgresSettings, + ) + from tipg.pg import close_db_connection as pg_close + from tipg.pg import connect_to_db as pg_connect + from tipg.pg import pg_get_collection_index + + db_settings = DatabaseSettings(schemas=["public"]) + catalog_settings = DuckDBCatalogSettings(tables=["main.points"]) + + @asynccontextmanager + async def lifespan(app: FastAPI): + await pg_connect( + app, + schemas=db_settings.schemas, + tipg_schema=db_settings.tipg_schema, + settings=PostgresSettings(database_url=database_url), + ) + await connect_to_db(app, settings=duckdb_settings) + + async with app.state.pool.acquire() as conn: + pg_collections = await pg_get_collection_index( + conn, settings=db_settings, connection_pool=app.state.pool + ) + duck_collections = await duck_get_collection_index( + app.state.duckdb_connection, catalog_settings + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*pg_collections, *duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + yield + + await close_db_connection(app) + await pg_close(app) + + app = FastAPI(lifespan=lifespan) + app.include_router(Endpoints(title="TiPg mixed").router) + add_exception_handlers(app, DEFAULT_STATUS_CODES) + + with TestClient(app) as client: + yield client diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py new file mode 100644 index 00000000..152c4607 --- /dev/null +++ b/tests/duck/test_duck.py @@ -0,0 +1,289 @@ +"""Test the DuckDB backend through the OGC API endpoints.""" + +import json + +import mapbox_vector_tile +import pytest +from geojson_pydantic import Feature, FeatureCollection + +from tipg.duck import DuckCollection + + +def test_collections(duck_app): + """List DuckDB tables, views and file sources.""" + response = duck_app.get("/collections") + assert response.status_code == 200 + ids = {c["id"] for c in response.json()["collections"]} + assert ids == { + "main.points", + "main.polygons", + "main.utm", + "main.nongeo", + "main.landmarks", + } + + catalog = duck_app.app.state.collection_catalog["collections"] + assert all(isinstance(c, DuckCollection) for c in catalog.values()) + assert catalog["main.landmarks"].type == "View" + assert catalog["main.points"].type == "Table" + + # the connection is not serialized + response = duck_app.get("/rawcatalog") + assert response.status_code == 200 + assert "connection" not in response.json()["collections"]["main.points"] + + +def test_collection_metadata(duck_app): + """Check extents, CRS and queryables.""" + body = duck_app.get("/collections/main.points").json() + assert body["extent"]["spatial"]["bbox"] == [[1.0, 1.0, 20.0, 20.0]] + intervals = body["extent"]["temporal"]["interval"] + # `day` (DATE) is the default datetime column (columns are sorted by name) + assert intervals[0] == ["2020-01-01T01:00:00+00:00", "2020-01-21T00:00:00+00:00"] + assert ["2020-01-01T01:00:00+00:00", "2020-01-01T20:00:00+00:00"] in intervals + + # projected table: extent is reprojected to WGS84 + body = duck_app.get("/collections/main.utm").json() + minx, miny, maxx, maxy = body["extent"]["spatial"]["bbox"][0] + assert -94 < minx < maxx < -92 + assert 44 < miny < maxy < 47 + utm = duck_app.app.state.collection_catalog["collections"]["main.utm"] + assert utm.geometry_column.srid == 26915 + assert utm.crs == "http://www.opengis.net/def/crs/EPSG/0/26915" + + body = duck_app.get("/collections/main.nongeo").json() + assert not body.get("extent") + + body = duck_app.get("/collections/main.points/queryables").json() + props = body["properties"] + assert props["geom"] == {"$ref": "https://geojson.org/schema/Point.json"} + assert props["id"]["type"] == "number" + assert props["price"]["type"] == "number" + assert props["name"]["type"] == "string" + assert props["tags"]["type"] == "array" + assert props["props"]["type"] == "object" + + +def test_collections_filters(duck_app): + """Filter collections by bbox / datetime (uses collection extents).""" + body = duck_app.get("/collections", params={"bbox": "0,0,5,5"}).json() + ids = {c["id"] for c in body["collections"]} + assert {"main.points", "main.landmarks"} <= ids + assert "main.utm" not in ids + + body = duck_app.get( + "/collections", params={"datetime": "2019-01-01T00:00:00Z/2019-12-31T00:00:00Z"} + ).json() + assert body["numberMatched"] == 0 + + +def test_items(duck_app): + """Get items.""" + response = duck_app.get("/collections/main.points/items") + assert response.status_code == 200 + body = response.json() + FeatureCollection.model_validate(body) + assert body["numberMatched"] == 20 + assert body["numberReturned"] == 10 + feat = body["features"][0] + assert feat["id"] == 1 + assert feat["geometry"] == {"type": "Point", "coordinates": [1.0, 1.0]} + assert feat["properties"]["name"] == "point 1" + assert feat["properties"]["tags"] == ["a", "b"] + assert feat["properties"]["props"] == {"rank": 1} + assert feat["properties"]["dt"] == "2020-01-01T01:00:00+00:00" + assert "geom" not in feat["properties"] + + # paging + body = duck_app.get( + "/collections/main.points/items", params={"limit": 5, "offset": 15} + ).json() + assert [f["id"] for f in body["features"]] == [16, 17, 18, 19, 20] + rels = {link["rel"] for link in body["links"]} + assert "prev" in rels + assert "next" not in rels + + # no primary key: `gid` fallback key + body = duck_app.get("/collections/main.polygons/items").json() + assert [f["id"] for f in body["features"]] == [1, 2, 3] + + # file source + body = duck_app.get("/collections/main.landmarks/items").json() + assert body["numberMatched"] == 5 + + # non spatial table + body = duck_app.get("/collections/main.nongeo/items").json() + assert body["numberMatched"] == 3 + assert body["features"][0]["geometry"] is None + + +def test_items_filters(duck_app): + """Filter items with ids / bbox / datetime / properties / cql2.""" + url = "/collections/main.points/items" + + body = duck_app.get(url, params={"ids": "2,4"}).json() + assert [f["id"] for f in body["features"]] == [2, 4] + + body = duck_app.get(url, params={"bbox": "0,0,3.5,3.5"}).json() + assert [f["id"] for f in body["features"]] == [1, 2, 3] + + body = duck_app.get( + url, + params={"datetime": "2020-01-01T03:00:00Z/..", "datetime-column": "dt"}, + ).json() + assert body["numberMatched"] == 18 + + body = duck_app.get(url, params={"name": "point 3"}).json() + assert [f["id"] for f in body["features"]] == [3] + + body = duck_app.get( + url, params={"filter": "id IN (5, 6, 7) AND name LIKE 'point %'"} + ).json() + assert [f["id"] for f in body["features"]] == [5, 6, 7] + + body = duck_app.get( + url, + params={ + "filter": "S_INTERSECTS(geom, POLYGON((0 0, 2.5 0, 2.5 2.5, 0 2.5, 0 0)))" + }, + ).json() + assert [f["id"] for f in body["features"]] == [1, 2] + + +def test_items_projected(duck_app): + """Filter and output items stored in EPSG:26915.""" + url = "/collections/main.utm/items" + + body = duck_app.get(url).json() + lon, lat = body["features"][0]["geometry"]["coordinates"] + assert -94 < lon < -93 and 44 < lat < 45.5 + + # bbox around Minneapolis only (CRS84) + body = duck_app.get(url, params={"bbox": "-94,44.5,-93,45.5"}).json() + assert [f["id"] for f in body["features"]] == [1] + + # cql2 geometries are CRS84 too + body = duck_app.get( + url, + params={ + "filter": "S_INTERSECTS(geom, POLYGON((-93 46, -91 46, -91 47.5, -93 47.5, -93 46)))" + }, + ).json() + assert [f["id"] for f in body["features"]] == [2] + + +def test_items_sortby_properties(duck_app): + """Sort and select properties.""" + url = "/collections/main.points/items" + + body = duck_app.get(url, params={"sortby": "-value", "limit": 3}).json() + assert [f["id"] for f in body["features"]] == [20, 19, 18] + + body = duck_app.get(url, params={"properties": "name", "limit": 1}).json() + assert body["features"][0]["properties"] == {"name": "point 1"} + + response = duck_app.get(url, params={"sortby": "nope"}) + assert response.status_code == 404 + + +def test_items_output_formats(duck_app): + """CSV / NDJSON outputs use the WKT geometry path.""" + url = "/collections/main.points/items" + + response = duck_app.get(url, params={"f": "csv", "limit": 2}) + assert response.status_code == 200 + rows = response.text.splitlines() + assert len(rows) == 3 + assert "SRID=4326;POINT (1 1)" in rows[1] + + response = duck_app.get(url, params={"f": "ndjson", "limit": 2}) + assert response.status_code == 200 + first = json.loads(response.text.splitlines()[0]) + assert first["geometry"] == "SRID=4326;POINT (1 1)" + + response = duck_app.get(url, params={"f": "geojsonseq", "limit": 2}) + assert response.status_code == 200 + Feature.model_validate(json.loads(response.text.splitlines()[0])) + + response = duck_app.get(url, params={"f": "html", "limit": 2}) + assert response.status_code == 200 + + +def test_item(duck_app): + """Get one item.""" + response = duck_app.get("/collections/main.points/items/3") + assert response.status_code == 200 + assert response.json()["id"] == 3 + + response = duck_app.get("/collections/main.points/items/999") + assert response.status_code == 404 + + # non-integer id on an integer key + response = duck_app.get("/collections/main.points/items/abc") + assert response.status_code == 404 + + +@pytest.mark.parametrize( + "collection,count", + [("main.points", 20), ("main.polygons", 3), ("main.utm", 2), ("main.landmarks", 5)], +) +def test_tiles(duck_app, collection, count): + """Get vector tiles.""" + response = duck_app.get(f"/collections/{collection}/tiles/WebMercatorQuad/0/0/0") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/vnd.mapbox-vector-tile" + decoded = mapbox_vector_tile.decode(response.content) + assert len(decoded["default"]["features"]) == count + + +def test_tiles_options(duck_app): + """Vector tiles properties, sortby, limit and empty tiles.""" + url = "/collections/main.points/tiles/WebMercatorQuad/0/0/0" + + decoded = mapbox_vector_tile.decode(duck_app.get(url).content) + props = decoded["default"]["features"][0]["properties"] + # types not supported by MVT are cast + assert props["price"] == 2.25 + assert props["day"] == "2020-01-02" + assert props["tags"] == "[a, b]" + + response = duck_app.get( + url, params={"sortby": "-id", "limit": 3, "properties": "id"} + ) + decoded = mapbox_vector_tile.decode(response.content) + assert [f["properties"]["id"] for f in decoded["default"]["features"]] == [ + 20, + 19, + 18, + ] + + response = duck_app.get(url, params={"sortby": "value", "properties": "id"}) + assert response.status_code == 404 + + # tile without data + response = duck_app.get("/collections/main.points/tiles/WebMercatorQuad/10/0/0") + assert response.status_code == 200 + decoded = mapbox_vector_tile.decode(response.content) + assert not decoded or not decoded["default"]["features"] + + +def test_mixed_catalog(mixed_app): + """Serve PostgreSQL and DuckDB collections from one catalog.""" + ids = {c["id"] for c in mixed_app.get("/collections").json()["collections"]} + assert "main.points" in ids + assert "public.landsat_wrs" in ids + + body = mixed_app.get("/collections/main.points/items", params={"limit": 1}).json() + assert body["numberMatched"] == 20 + + body = mixed_app.get( + "/collections/public.landsat_wrs/items", params={"limit": 1} + ).json() + assert body["numberMatched"] == 16269 + + response = mixed_app.get("/collections/main.points/tiles/WebMercatorQuad/0/0/0") + assert response.status_code == 200 + response = mixed_app.get( + "/collections/public.landsat_wrs/tiles/WebMercatorQuad/0/0/0" + ) + assert response.status_code == 200 From e375e2809c79ba29dafe1b8d31896f556a4f7161 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:14:42 +0100 Subject: [PATCH 07/22] docs: add DuckDB backend guide and API pages Co-Authored-By: Claude Opus 5.5 --- docs/mkdocs.yml | 5 + docs/src/advanced/duckdb.md | 165 ++++++++++++++++++++++++++ docs/src/api/tipg/duck/collections.md | 1 + docs/src/api/tipg/duck/database.md | 1 + docs/src/api/tipg/duck/settings.md | 1 + 5 files changed, 173 insertions(+) create mode 100644 docs/src/advanced/duckdb.md create mode 100644 docs/src/api/tipg/duck/collections.md create mode 100644 docs/src/api/tipg/duck/database.md create mode 100644 docs/src/api/tipg/duck/settings.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4e8f6465..8f7cdfa5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -27,11 +27,16 @@ nav: - "Customization": advanced/customization.md - "TiPg Vector Tiles server": advanced/ogc_tiles_server.md - "TiPg Features server": advanced/ogc_features_server.md + - "DuckDB backend": advanced/duckdb.md - API: - pg: - collections: api/tipg/pg/collections.md - database: api/tipg/pg/database.md - settings: api/tipg/pg/settings.md + - duck: + - collections: api/tipg/duck/collections.md + - database: api/tipg/duck/database.md + - settings: api/tipg/duck/settings.md - collections: api/tipg/collections.md - dependencies: api/tipg/dependencies.md - errors: api/tipg/errors.md diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md new file mode 100644 index 00000000..2d7317d2 --- /dev/null +++ b/docs/src/advanced/duckdb.md @@ -0,0 +1,165 @@ + +`TiPg` can serve tables, views and files (GeoParquet, or any format supported by DuckDB's `ST_Read`) through [DuckDB](https://duckdb.org) and its [spatial extension](https://duckdb.org/docs/extensions/spatial/overview), alongside or instead of PostgreSQL/PostGIS. + +!!! note + + The DuckDB backend is not used by the default `tipg.main:app` application: you need to create your own application (see examples below). + +### Install + +```bash +python -m pip install "tipg[duckdb]" +``` + +### Settings + +`tipg.duck.DuckDBSettings` (connection) and `tipg.duck.DuckDBCatalogSettings` (catalog) can be set with `TIPG_DUCKDB_` prefixed environment variables. + +| Setting | Env | Default | Description | +| ------- | --- | ------- | ----------- | +| `database` | `TIPG_DUCKDB_DATABASE` | `:memory:` | Path to a DuckDB database file, attached under its file name (without extension) | +| `read_only` | `TIPG_DUCKDB_READ_ONLY` | `True` | Attach the database file in read-only mode | +| `threads` | `TIPG_DUCKDB_THREADS` | | Number of threads DuckDB can use | +| `memory_limit` | `TIPG_DUCKDB_MEMORY_LIMIT` | | Maximum memory DuckDB can use (e.g `4GB`) | +| `sources` | `TIPG_DUCKDB_SOURCES` | `{}` | Mapping of name to file path/glob/URL (e.g `'{"countries": "data/countries.parquet"}'`). Parquet files are read with `read_parquet`, other formats with `ST_Read` | +| `schemas` | `TIPG_DUCKDB_SCHEMAS` | `["main"]` | Schemas to look for tables/views in | +| `tables` | `TIPG_DUCKDB_TABLES` | | Only expose those tables (`schema.table`) | +| `exclude_tables` | `TIPG_DUCKDB_EXCLUDE_TABLES` | | Tables (`schema.table`) to exclude | +| `only_spatial_tables` | `TIPG_DUCKDB_ONLY_SPATIAL_TABLES` | `True` | Only expose tables/views with a geometry column | +| `spatial_extent` | `TIPG_DUCKDB_SPATIAL_EXTENT` | `True` | Compute the spatial extent of each collection | +| `datetime_extent` | `TIPG_DUCKDB_DATETIME_EXTENT` | `True` | Compute the temporal extent of each collection | + +Table level configuration (`TIPG_TABLE_CONFIG__...`, fallback key names, column sorting) is shared with the PostgreSQL backend (see [Configuration](../user_guide/configuration.md)). + +Collections are identified by `{schema}.{table}`; file `sources` are registered as views in the in-memory `main` schema, so a `countries` source is exposed as the `main.countries` collection. + +### DuckDB application + +```python +"""TiPg DuckDB application.""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + DuckDBSettings, + close_db_connection, + connect_to_db, + duck_get_collection_index, +) +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.factory import Endpoints +from tipg.middleware import CatalogUpdateMiddleware + +duckdb_settings = DuckDBSettings() +catalog_settings = DuckDBCatalogSettings() + + +async def register_collection_catalog(app: FastAPI) -> None: + """Register Collection catalog.""" + collections = await duck_get_collection_index( + app.state.duckdb_connection, + catalog_settings, + ) + app.state.collection_catalog = Catalog( + collections={col.id: col for col in collections}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + await connect_to_db(app, settings=duckdb_settings) + await register_collection_catalog(app) + yield + await close_db_connection(app) + + +app = FastAPI(title="TiPg DuckDB", lifespan=lifespan) + +ogc_api = Endpoints(title="TiPg DuckDB") +app.include_router(ogc_api.router) + +app.add_middleware(CatalogUpdateMiddleware, func=register_collection_catalog, ttl=300) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) +``` + +```bash +TIPG_DUCKDB_DATABASE=data.duckdb \ +TIPG_DUCKDB_SOURCES='{"countries": "countries.parquet"}' \ +uvicorn app:app --port 8000 +``` + +### PostgreSQL and DuckDB in a single application + +Each collection carries its own connection (`PgCollection.connection_pool`, `DuckCollection.connection`), so collections from both backends can be served by the same endpoints. Merge the collections from both backends in your `register_collection_catalog` function: + +```python +"""TiPg PostgreSQL + DuckDB application.""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from tipg import duck, pg +from tipg.collections import Catalog +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.factory import Endpoints + +db_settings = pg.DatabaseSettings() + + +async def register_collection_catalog(app: FastAPI) -> None: + """Register Collection catalog.""" + async with app.state.pool.acquire() as conn: + pg_collections = await pg.pg_get_collection_index( + conn, + settings=db_settings, + connection_pool=app.state.pool, + ) + + duck_collections = await duck.duck_get_collection_index(app.state.duckdb_connection) + + # NOTE: collections are identified by `{schema}.{table}`, make sure + # both backends don't expose the same ids. + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*pg_collections, *duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + await pg.connect_to_db( + app, + schemas=db_settings.schemas, + tipg_schema=db_settings.tipg_schema, + settings=pg.PostgresSettings(), + ) + await duck.connect_to_db(app, settings=duck.DuckDBSettings()) + await register_collection_catalog(app) + yield + await duck.close_db_connection(app) + await pg.close_db_connection(app) + + +app = FastAPI(title="TiPg", lifespan=lifespan) +app.include_router(Endpoints(title="TiPg").router) +add_exception_handlers(app, DEFAULT_STATUS_CODES) +``` + +### Differences with the PostgreSQL backend + +- **CRS**: the SRID of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. +- **Spatial extent**: computed with a full scan of the geometry column (DuckDB has no equivalent of PostGIS' `ST_EstimatedExtent`). Set `TIPG_DUCKDB_SPATIAL_EXTENT=FALSE` for large datasets. +- **Vector tiles**: `ST_AsMVT` only supports boolean, integer, float and string properties; `DECIMAL` values are cast to `DOUBLE` and other types (dates, timestamps, lists, ...) to `VARCHAR`. +- **Functions**: SQL function layers are not supported. +- **WKT output** (CSV, (nd)json): DuckDB's `ST_AsText` formatting (e.g `SRID=4326;POINT (1 2)`). diff --git a/docs/src/api/tipg/duck/collections.md b/docs/src/api/tipg/duck/collections.md new file mode 100644 index 00000000..13d62d68 --- /dev/null +++ b/docs/src/api/tipg/duck/collections.md @@ -0,0 +1 @@ +::: tipg.duck.collections diff --git a/docs/src/api/tipg/duck/database.md b/docs/src/api/tipg/duck/database.md new file mode 100644 index 00000000..7508272e --- /dev/null +++ b/docs/src/api/tipg/duck/database.md @@ -0,0 +1 @@ +::: tipg.duck.database diff --git a/docs/src/api/tipg/duck/settings.md b/docs/src/api/tipg/duck/settings.md new file mode 100644 index 00000000..fdd22381 --- /dev/null +++ b/docs/src/api/tipg/duck/settings.md @@ -0,0 +1 @@ +::: tipg.duck.settings From 47f8ff0c3da766b656fe5aa085968e5335240375 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:16:20 +0100 Subject: [PATCH 08/22] docs: add DuckDB backend implementation plan Co-Authored-By: Claude Opus 5.5 --- plans/duckdb-backend.md | 83 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 plans/duckdb-backend.md diff --git a/plans/duckdb-backend.md b/plans/duckdb-backend.md new file mode 100644 index 00000000..a8b1d066 --- /dev/null +++ b/plans/duckdb-backend.md @@ -0,0 +1,83 @@ +# DuckDB backend for tipg (`tipg.duck`) + +## Goal +Serve DuckDB tables/views and files (GeoParquet, `ST_Read`-able files) as OGC Features + Tiles collections, and allow mixing them with PostgreSQL collections in a single catalog. + +Scope: DuckDB database file + file sources; features + MVT tiles; no app module (`tipg/main.py` untouched). + +## DuckDB constraints the design relies on (DuckDB 2.0 + spatial) +- cql2 `to_sql()` output runs as-is (`= SOME(ARRAY[...])`, timestamp casts, `LIKE .. ESCAPE`, `st_intersects(.., st_geomfromgeojson(..))`). +- `ST_AsMVT(t, 'layer' ORDER BY ..)` / `ST_AsMVTGeom(geom, BOX_2D, extent, buffer, clip)` are available. +- Geometry columns carry their CRS in the type (`GEOMETRY('EPSG:26915')`, `GEOMETRY('OGC:CRS84')`, or plain `GEOMETRY`); `ST_CRS` / `ST_SetCRS` exist. +- Comparing geometries in different CRS silently returns no match → filter literals must be reprojected/tagged. +- `SET geometry_always_xy = true` is required for lon/lat axis order; `ST_Transform(g, 'EPSG:a', 'EPSG:b')` (or a PROJ string) then works positionally. Use `EPSG:4326` (not `OGC:CRS84`) as output CRS. +- `ST_AsGeoJSON` returns a JSON string; `ST_Extent_Agg` returns a GEOMETRY; no `ST_EstimatedExtent`. +- Primary keys via `duckdb_constraints()`. + +To confirm at the start of implementation: MVT with TIMESTAMP/DATE/LIST/JSON/DECIMAL properties (cast to VARCHAR if unsupported), empty-tile result (NULL → `b""`), TIMESTAMPTZ values in Python with `SET TimeZone = 'UTC'`, TEMP views in `duckdb_views()` / `duckdb_columns()`, CRS reported by `ST_Read`, cql2 emission of `{"op": "st_transform", "args": [geom, "EPSG:4326", "EPSG:n"]}`. + +## Changes + +### Setup +- Rebase `feat/duckdb-backend` on `main`. +- `pyproject.toml`: `duckdb = ["duckdb>=1.4"]` optional extra; add `tipg[duckdb]` to the dev group; `uv sync`. + +### `tipg/collections.py` — two overridable hooks (PG output unchanged) +`cql_where()` hard-codes PG constructs (`st_setsrid(polygon, srid)` in `_s_intersects_bbox`, `st_transform(literal, srid)` in `_transform_cql_geom_literals`). Route them through methods on `Collection` whose defaults keep today's PG output: +```python +def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: # default: st_setsrid(geom, srid) +def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: # default: st_transform(geom, srid) +``` +`DuckCollection` overrides with `st_setcrs(geom, 'EPSG:n')` and `st_transform(geom, 'EPSG:4326', 'EPSG:n')`. +Move `_quote_ident` from `tipg/pg/collections.py` to `tipg/collections.py` (same quoting rules in DuckDB). + +### `tipg/duck/settings.py` (new) +- `DuckDBSettings` (`TIPG_DUCKDB_`): `database: str = ":memory:"`, `read_only: bool = True`, `threads: int | None`, `memory_limit: str | None`, `sources: dict[str, str] = {}` (collection id → path/glob; parquet via `read_parquet`, others via `ST_Read`). +- `DuckDBCatalogSettings` (`TIPG_DUCKDB_`): `schemas = ["main"]`, `tables`, `exclude_tables`, `only_spatial_tables = True`, `spatial_extent = True`, `datetime_extent = True`. + +### `tipg/duck/database.py` (new) +- `connect_to_db(app, *, settings=None)`: `duckdb.connect(database, read_only)`, `INSTALL/LOAD spatial`, `SET geometry_always_xy = true`, `SET TimeZone = 'UTC'`, threads / memory_limit, register each source as `CREATE TEMP VIEW "" AS SELECT * FROM ...`; store on `app.state.duckdb_connection`. +- `close_db_connection(app)`. + +### `tipg/duck/collections.py` (new) +`DuckCollection(Collection)`: +- `connection: duckdb.DuckDBPyConnection = Field(exclude=True, repr=False)` (`arbitrary_types_allowed`), plus the fully qualified relation name (`"db"."schema"."table"`). +- `features()` / `get_tile()`: same signatures and validation as `PgCollection` (`InvalidGeometryColumnName`, `InvalidLimit`, sort-column check), same SQL-builder structure (`_select`, `_select_mvt`, `_from`, `_sort_expressions`, `_where_clause`). Queries run in `starlette.concurrency.run_in_threadpool` on a per-call `self.connection.cursor()` (closed after); count + page share the cursor. +- Geometry output: `ST_Transform(g, 'EPSG:', 'EPSG:4326')` when srid ≠ 4326; `ST_Envelope` for bbox-only; simplify = `ST_ReducePrecision(ST_SimplifyPreserveTopology(g, s), s)`; GeoJSON via `ST_AsGeoJSON` → `orjson.loads`; WKT = `'SRID=4326;' || ST_AsText(g)`; JSON-typed property values decoded with `orjson.loads`. +- Id: primary-key column, else `row_number() OVER ()`. +- MVT: inner `SELECT props, ST_AsMVTGeom(, ST_Extent(ST_MakeEnvelope()), res, buffer, clip) AS geom ... WHERE .. ORDER BY .. LIMIT n`; outer `SELECT ST_AsMVT(t, $1 [ORDER BY ..]) FROM t`. Out-of-domain tiles clipped with `ST_Intersection(ST_MakeEnvelope(tms.bbox), ST_Transform(g, src, 'EPSG:'))` like PG; TMS CRS as `EPSG:n` or PROJ string; NULL tile → `b""`. +- No function collections: `type` is `Table` / `View`; `function_parameters` accepted and ignored. + +`async duck_get_collection_index(connection, settings=None) -> list[Collection]` (blocking work in the threadpool): +- Tables/views of `current_database()` in `settings.schemas` (+ include/exclude filters), plus the TEMP source views. Ids: `schema.table`; sources use their key. +- Columns from `duckdb_columns()`, types normalized to the names `Column` already understands (`GEOMETRY*→geometry`, `TIMESTAMP→timestamp`, `TIMESTAMP WITH TIME ZONE→timestamptz`, `DATE→date`, integer types, `DOUBLE→float8`, `FLOAT→real`, `DECIMAL(..)→numeric`, `BOOLEAN→boolean`, `JSON→json`, `X[]→x[]`, else `text`) so `json_type`, `is_geometry`, `is_datetime` and `_coerce_id` work unchanged. +- Geometry: srid parsed from `GEOMETRY('EPSG:n')` (`OGC:CRS84` / no CRS → 4326); when `spatial_extent`, one scan per table for `ST_Extent_Agg` (→ 4326 if needed) and `min/max(ST_GeometryType)` (single type, else `"Geometry"`). +- Datetime extents (when `datetime_extent`): one `min/max` query per table covering all datetime columns, ISO strings matching PG output (naive timestamps as UTC, dates as midnight UTC). +- id / geometry / datetime column selection with `TableConfig` / `TableSettings` from `tipg.settings` (`fallback_key_names`, `sort_columns`, `table_config` keyed `_`, sources keyed by id), same logic as `pg_get_collection_index`. +- `only_spatial_tables` filter. + +### `tipg/duck/__init__.py` +Export `connect_to_db`, `close_db_connection`, `duck_get_collection_index`, `DuckCollection`, `DuckDBSettings`, `DuckDBCatalogSettings`. + +### Docs / changelog +- `docs/src/api/tipg/duck/{collections,database,settings}.md` + mkdocs nav. +- User-guide page: a DuckDB-only app and a mixed PG + DuckDB app (`register_collection_catalog` merging both index lists). +- `CHANGES.md` entries. + +## Tests (`tests/duck/`) +- `conftest.py`: session-scoped temp `.duckdb` built via SQL — points (INTEGER PK, TIMESTAMPTZ, DATE, text / double / list props), polygons without PK (`gid` fallback key), a `GEOMETRY('EPSG:26915')` table, a non-spatial table — plus a GeoParquet file declared as a source. App fixture: FastAPI + `Endpoints` + lifespan with a local `register_collection_catalog` (pattern of `tests/conftest.py:create_tipg_app`). A second fixture builds a mixed PG + DuckDB app. +- Coverage: `/collections` (extents, bbox / datetime filters), `/queryables`, `/items` with `ids`, `bbox`, `datetime`, cql2 `filter` (`IN`, `LIKE`, `S_INTERSECTS`, including on the EPSG:26915 table), `sortby`, `properties`, `limit` / `offset` + links, csv / ndjson (WKT path), `/items/{id}` (bad int id → 404), MVT (feature count, layer name, sortby order, empty tile), and the mixed app serving both a PG and a DuckDB collection. + +## Verification +1. Confirm the open DuckDB points listed above; adjust casts if needed. +2. `uv run pytest tests/duck -v`, then the full `uv run pytest` (PG suite guards the `cql_where` hook refactor). +3. `uv run pre-commit run --all-files`. +4. Manual: small app over the test `.duckdb` with uvicorn; check `/collections//items?f=html` and the tile viewer. + +## Implementation notes (deviations from the plan above) +- **Open points confirmed**: cql2 emits `st_transform(g, 'EPSG:4326', 'EPSG:n')` / `st_setcrs(g, 'EPSG:n')` as expected; `ST_Read` reports the CRS in the column type; an empty tile is a valid MVT with an empty layer (not NULL). +- **`pytz`**: DuckDB needs it to return `TIMESTAMP WITH TIME ZONE` values to Python → added to the `duckdb` extra. +- **Connection layout**: `connection.cursor()` opens a new connection, which can't see `TEMP` views or session-level `SET`. The connection now always uses an in-memory primary database; `database` (file) is `ATTACH`ed (read-only by default) under its file name, sources are regular views in `memory.main`, and settings use `SET GLOBAL`. +- **Source ids**: sources are exposed as `main.` (same `schema.table` rule as tables) instead of their bare key; duplicate ids across attached databases are skipped with a warning. +- **MVT property casts**: `ST_AsMVT` supports BOOLEAN, integer types (not HUGEINT), FLOAT/DOUBLE, VARCHAR and JSON; DECIMAL/HUGEINT/UBIGINT are cast to DOUBLE, other types (dates, timestamps, lists, UUID, …) to VARCHAR. +- **Tests**: `tests/duck/conftest.py` imports `tipg.factory` inside fixtures; a module-level import read the module-level settings before the PG fixtures set their environment and broke 4 PG tests. From 865d8cbe6fa4c3cbef6cac45a261959400c47041 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:47:33 +0100 Subject: [PATCH 09/22] feat(duck): make the DuckDB database file optional DuckDBSettings.database now defaults to None: the database file (path or URL) is only attached when set, otherwise only file sources are served. Co-Authored-By: Claude Opus 5.5 --- docs/src/advanced/duckdb.md | 2 +- tests/duck/conftest.py | 10 ++++++++++ tests/duck/test_duck.py | 23 +++++++++++++++++++++++ tipg/duck/database.py | 10 ++++++---- tipg/duck/settings.py | 6 +++--- 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index 2d7317d2..8913668f 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -17,7 +17,7 @@ python -m pip install "tipg[duckdb]" | Setting | Env | Default | Description | | ------- | --- | ------- | ----------- | -| `database` | `TIPG_DUCKDB_DATABASE` | `:memory:` | Path to a DuckDB database file, attached under its file name (without extension) | +| `database` | `TIPG_DUCKDB_DATABASE` | | Path or URL to a DuckDB database file, attached under its file name (without extension). If not set, only `sources` are served | | `read_only` | `TIPG_DUCKDB_READ_ONLY` | `True` | Attach the database file in read-only mode | | `threads` | `TIPG_DUCKDB_THREADS` | | Number of threads DuckDB can use | | `memory_limit` | `TIPG_DUCKDB_MEMORY_LIMIT` | | Maximum memory DuckDB can use (e.g `4GB`) | diff --git a/tests/duck/conftest.py b/tests/duck/conftest.py index 074b2f15..b5e4d51f 100644 --- a/tests/duck/conftest.py +++ b/tests/duck/conftest.py @@ -186,3 +186,13 @@ async def lifespan(app: FastAPI): with TestClient(app) as client: yield client + + +@pytest.fixture +def duck_app_sources_only(duckdb_files): + """DuckDB TiPg app without database file (file sources only).""" + settings = DuckDBSettings(sources={"landmarks": duckdb_files["parquet"]}) + assert settings.database is None + app = create_duck_app(settings, DuckDBCatalogSettings()) + with TestClient(app) as client: + yield client diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py index 152c4607..ad3ce071 100644 --- a/tests/duck/test_duck.py +++ b/tests/duck/test_duck.py @@ -287,3 +287,26 @@ def test_mixed_catalog(mixed_app): "/collections/public.landsat_wrs/tiles/WebMercatorQuad/0/0/0" ) assert response.status_code == 200 + + +def test_sources_only(duck_app_sources_only): + """Without database file, only the file sources are served.""" + body = duck_app_sources_only.get("/collections").json() + assert [c["id"] for c in body["collections"]] == ["main.landmarks"] + + body = duck_app_sources_only.get("/collections/main.landmarks/items").json() + assert body["numberMatched"] == 5 + + +def test_database_alias(duckdb_files): + """The database file is attached under its file name, also for URLs.""" + from tipg.duck import DuckDBSettings, create_connection + + conn = create_connection( + DuckDBSettings(database=f"file://{duckdb_files['database']}") + ) + rows = conn.execute( + "SELECT database_name, readonly FROM duckdb_databases() WHERE NOT internal" + ).fetchall() + assert ("test", True) in rows + conn.close() diff --git a/tipg/duck/database.py b/tipg/duck/database.py index bc924b5e..4f6d29f9 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -1,6 +1,7 @@ """tipg DuckDB database events.""" import pathlib +from urllib.parse import urlparse import duckdb from fastapi import FastAPI @@ -29,8 +30,8 @@ def create_connection( """Create and configure a DuckDB connection. The connection always uses an in-memory primary database, where file - `sources` are registered as views. When `settings.database` points to a - database file, it is attached (read-only by default) under its file name + `sources` are registered as views. When `settings.database` is set, the + database file is attached (read-only by default) under its file name (without extension). Note: views (and settings) must be visible from `connection.cursor()`, @@ -54,8 +55,9 @@ def create_connection( conn.execute("SET GLOBAL geometry_always_xy = true") conn.execute("SET GLOBAL TimeZone = 'UTC'") - if settings.database not in ("", ":memory:"): - alias = pathlib.Path(settings.database).stem + if settings.database: + # use the file name (without extension) as alias, also for URLs + alias = pathlib.PurePosixPath(urlparse(settings.database).path).stem options = " (READ_ONLY)" if settings.read_only else "" logger.debug(f"Attaching DuckDB database {settings.database} as {alias}") conn.execute( diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py index 6d84df4f..497ce4d0 100644 --- a/tipg/duck/settings.py +++ b/tipg/duck/settings.py @@ -7,8 +7,8 @@ class DuckDBSettings(BaseSettings): """DuckDB connection settings. Attributes: - database: path to a DuckDB database file, attached under its file name - (default to an in-memory database only). + database: path or URL to a DuckDB database file, attached under its + file name (without extension). If not set, only `sources` are served. read_only: attach the database file in read-only mode. threads: number of threads DuckDB can use. memory_limit: maximum memory DuckDB can use (e.g `4GB`). @@ -19,7 +19,7 @@ class DuckDBSettings(BaseSettings): """ - database: str = ":memory:" + database: str | None = None read_only: bool = True threads: int | None = None memory_limit: str | None = None From 88c1c85cbf400ceaf40ae69e7e8b2714fcfcaee6 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 11:49:07 +0100 Subject: [PATCH 10/22] feat(duck): always attach the DuckDB database read-only Remove the read_only setting: tipg only reads data. Co-Authored-By: Claude Opus 5.5 --- docs/src/advanced/duckdb.md | 3 +-- plans/duckdb-backend.md | 2 +- tipg/duck/database.py | 5 ++--- tipg/duck/settings.py | 7 +++---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index 8913668f..d30e4851 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -17,8 +17,7 @@ python -m pip install "tipg[duckdb]" | Setting | Env | Default | Description | | ------- | --- | ------- | ----------- | -| `database` | `TIPG_DUCKDB_DATABASE` | | Path or URL to a DuckDB database file, attached under its file name (without extension). If not set, only `sources` are served | -| `read_only` | `TIPG_DUCKDB_READ_ONLY` | `True` | Attach the database file in read-only mode | +| `database` | `TIPG_DUCKDB_DATABASE` | | Path or URL to a DuckDB database file, attached (read-only) under its file name (without extension). If not set, only `sources` are served | | `threads` | `TIPG_DUCKDB_THREADS` | | Number of threads DuckDB can use | | `memory_limit` | `TIPG_DUCKDB_MEMORY_LIMIT` | | Maximum memory DuckDB can use (e.g `4GB`) | | `sources` | `TIPG_DUCKDB_SOURCES` | `{}` | Mapping of name to file path/glob/URL (e.g `'{"countries": "data/countries.parquet"}'`). Parquet files are read with `read_parquet`, other formats with `ST_Read` | diff --git a/plans/duckdb-backend.md b/plans/duckdb-backend.md index a8b1d066..f73224a3 100644 --- a/plans/duckdb-backend.md +++ b/plans/duckdb-backend.md @@ -77,7 +77,7 @@ Export `connect_to_db`, `close_db_connection`, `duck_get_collection_index`, `Duc ## Implementation notes (deviations from the plan above) - **Open points confirmed**: cql2 emits `st_transform(g, 'EPSG:4326', 'EPSG:n')` / `st_setcrs(g, 'EPSG:n')` as expected; `ST_Read` reports the CRS in the column type; an empty tile is a valid MVT with an empty layer (not NULL). - **`pytz`**: DuckDB needs it to return `TIMESTAMP WITH TIME ZONE` values to Python → added to the `duckdb` extra. -- **Connection layout**: `connection.cursor()` opens a new connection, which can't see `TEMP` views or session-level `SET`. The connection now always uses an in-memory primary database; `database` (file) is `ATTACH`ed (read-only by default) under its file name, sources are regular views in `memory.main`, and settings use `SET GLOBAL`. +- **Connection layout**: `connection.cursor()` opens a new connection, which can't see `TEMP` views or session-level `SET`. The connection now always uses an in-memory primary database; `database` (optional file path/URL) is `ATTACH`ed read-only under its file name (no `read_only` setting), sources are regular views in `memory.main`, and settings use `SET GLOBAL`. - **Source ids**: sources are exposed as `main.` (same `schema.table` rule as tables) instead of their bare key; duplicate ids across attached databases are skipped with a warning. - **MVT property casts**: `ST_AsMVT` supports BOOLEAN, integer types (not HUGEINT), FLOAT/DOUBLE, VARCHAR and JSON; DECIMAL/HUGEINT/UBIGINT are cast to DOUBLE, other types (dates, timestamps, lists, UUID, …) to VARCHAR. - **Tests**: `tests/duck/conftest.py` imports `tipg.factory` inside fixtures; a module-level import read the module-level settings before the PG fixtures set their environment and broke 4 PG tests. diff --git a/tipg/duck/database.py b/tipg/duck/database.py index 4f6d29f9..5642a014 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -31,7 +31,7 @@ def create_connection( The connection always uses an in-memory primary database, where file `sources` are registered as views. When `settings.database` is set, the - database file is attached (read-only by default) under its file name + database file is attached, read-only, under its file name (without extension). Note: views (and settings) must be visible from `connection.cursor()`, @@ -58,10 +58,9 @@ def create_connection( if settings.database: # use the file name (without extension) as alias, also for URLs alias = pathlib.PurePosixPath(urlparse(settings.database).path).stem - options = " (READ_ONLY)" if settings.read_only else "" logger.debug(f"Attaching DuckDB database {settings.database} as {alias}") conn.execute( - f"ATTACH {_quote_literal(settings.database)} AS {_quote_ident(alias)}{options}" + f"ATTACH {_quote_literal(settings.database)} AS {_quote_ident(alias)} (READ_ONLY)" ) for name, path in settings.sources.items(): diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py index 497ce4d0..1fa610e1 100644 --- a/tipg/duck/settings.py +++ b/tipg/duck/settings.py @@ -7,9 +7,9 @@ class DuckDBSettings(BaseSettings): """DuckDB connection settings. Attributes: - database: path or URL to a DuckDB database file, attached under its - file name (without extension). If not set, only `sources` are served. - read_only: attach the database file in read-only mode. + database: path or URL to a DuckDB database file, attached (read-only) + under its file name (without extension). If not set, only `sources` + are served. threads: number of threads DuckDB can use. memory_limit: maximum memory DuckDB can use (e.g `4GB`). sources: mapping of collection id to a file path/glob/URL. Parquet files @@ -20,7 +20,6 @@ class DuckDBSettings(BaseSettings): """ database: str | None = None - read_only: bool = True threads: int | None = None memory_limit: str | None = None sources: dict[str, str] = {} From dfafbf1072f562a209d6d586a111a651b49fddd6 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 12:11:00 +0100 Subject: [PATCH 11/22] chore: simplify --- tipg/duck/collections.py | 4 ++-- tipg/duck/database.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 20b20995..55198393 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -310,7 +310,7 @@ async def features( bbox_only: bool | None = None, simplify: float | None = None, geom_as_wkt: bool = False, - function_parameters: dict[str, str] | None = None, + **kwargs: Any, ) -> ItemList: """Build and run DuckDB query.""" limit = limit or features_settings.default_features_limit @@ -374,7 +374,6 @@ async def get_tile( bbox_filter: list[float] | None = None, datetime_filter: list[str] | None = None, properties_filter: list[tuple[str, str]] | None = None, - function_parameters: dict[str, str] | None = None, cql_filter: Expr | None = None, sortby: str | None = None, properties: list[str] | None = None, @@ -382,6 +381,7 @@ async def get_tile( dt: str | None = None, limit: int | None = None, simplify: float | None = None, + **kwargs: Any, ) -> bytes: """Build query to get Vector Tile.""" limit = limit or mvt_settings.max_features_per_tile diff --git a/tipg/duck/database.py b/tipg/duck/database.py index 5642a014..519211eb 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -24,9 +24,7 @@ def _source_query(path: str) -> str: return f"SELECT * FROM ST_Read({_quote_literal(path)})" -def create_connection( - settings: DuckDBSettings | None = None, -) -> duckdb.DuckDBPyConnection: +def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: """Create and configure a DuckDB connection. The connection always uses an in-memory primary database, where file @@ -38,8 +36,6 @@ def create_connection( which creates a new connection to the same database instance, so we can't use TEMP views (connection-local) nor session-level `SET`. """ - settings = settings or DuckDBSettings() - config: dict[str, str | int] = {} if settings.threads: config["threads"] = settings.threads @@ -55,6 +51,7 @@ def create_connection( conn.execute("SET GLOBAL geometry_always_xy = true") conn.execute("SET GLOBAL TimeZone = 'UTC'") + # Attach the main database file if specified in settings. if settings.database: # use the file name (without extension) as alias, also for URLs alias = pathlib.PurePosixPath(urlparse(settings.database).path).stem @@ -63,6 +60,7 @@ def create_connection( f"ATTACH {_quote_literal(settings.database)} AS {_quote_ident(alias)} (READ_ONLY)" ) + # Attach and register file sources as views in the in-memory database. for name, path in settings.sources.items(): logger.debug(f"Registering DuckDB source {name}: {path}") conn.execute( @@ -78,6 +76,9 @@ async def connect_to_db( settings: DuckDBSettings | None = None, ) -> None: """Connect to DuckDB and store the connection in `app.state.duckdb_connection`.""" + if not settings: + settings = DuckDBSettings() + app.state.duckdb_connection = create_connection(settings) From 4e07c6e075a7e5cc559d701b40649f09934b2279 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 12:13:28 +0100 Subject: [PATCH 12/22] fix(duck): match duckdb.connect config type Co-Authored-By: Claude Opus 5.5 --- tipg/duck/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tipg/duck/database.py b/tipg/duck/database.py index 519211eb..8fd368ec 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -36,7 +36,7 @@ def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: which creates a new connection to the same database instance, so we can't use TEMP views (connection-local) nor session-level `SET`. """ - config: dict[str, str | int] = {} + config: dict[str, str | bool | int | float | list[str]] = {} if settings.threads: config["threads"] = settings.threads if settings.memory_limit: From 54c359e69c6c9705bdfeadb752e4b7f93c9f70a1 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 15:20:55 +0100 Subject: [PATCH 13/22] refactor(duck): rename connection functions and make duck_get_collection_index keyword-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit connect_to_db / close_db_connection → connect_to_duckdb / close_duckdb_connection, duck_get_collection_index(*, connection, settings=None); update docs and tests. Co-Authored-By: Claude Opus 5.5 --- docs/src/advanced/duckdb.md | 36 +++++++++++++++--------------- tests/duck/conftest.py | 44 +++++++++++++++++++------------------ tipg/duck/__init__.py | 6 ++--- tipg/duck/collections.py | 1 + tipg/duck/database.py | 4 ++-- 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index d30e4851..ffce5501 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -46,8 +46,8 @@ from tipg.collections import Catalog from tipg.duck import ( DuckDBCatalogSettings, DuckDBSettings, - close_db_connection, - connect_to_db, + close_duckdb_connection, + connect_to_duckdb, duck_get_collection_index, ) from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers @@ -61,8 +61,8 @@ catalog_settings = DuckDBCatalogSettings() async def register_collection_catalog(app: FastAPI) -> None: """Register Collection catalog.""" collections = await duck_get_collection_index( - app.state.duckdb_connection, - catalog_settings, + connection=app.state.duckdb_connection, + settings=catalog_settings, ) app.state.collection_catalog = Catalog( collections={col.id: col for col in collections}, @@ -73,10 +73,10 @@ async def register_collection_catalog(app: FastAPI) -> None: @asynccontextmanager async def lifespan(app: FastAPI): """FastAPI Lifespan.""" - await connect_to_db(app, settings=duckdb_settings) + await connect_to_duckdb(app, settings=duckdb_settings) await register_collection_catalog(app) yield - await close_db_connection(app) + await close_duckdb_connection(app) app = FastAPI(title="TiPg DuckDB", lifespan=lifespan) @@ -112,19 +112,19 @@ from tipg.collections import Catalog from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers from tipg.factory import Endpoints -db_settings = pg.DatabaseSettings() +pg_catalog_settings = pg.PostgresCatalogSettings() async def register_collection_catalog(app: FastAPI) -> None: """Register Collection catalog.""" - async with app.state.pool.acquire() as conn: - pg_collections = await pg.pg_get_collection_index( - conn, - settings=db_settings, - connection_pool=app.state.pool, - ) + pg_collections = await pg.pg_get_collection_index( + connection_pool=app.state.pool, + settings=pg_catalog_settings, + ) - duck_collections = await duck.duck_get_collection_index(app.state.duckdb_connection) + duck_collections = await duck.duck_get_collection_index( + connection=app.state.duckdb_connection, + ) # NOTE: collections are identified by `{schema}.{table}`, make sure # both backends don't expose the same ids. @@ -139,14 +139,14 @@ async def lifespan(app: FastAPI): """FastAPI Lifespan.""" await pg.connect_to_db( app, - schemas=db_settings.schemas, - tipg_schema=db_settings.tipg_schema, + schemas=pg_catalog_settings.schemas, + tipg_schema=pg_catalog_settings.tipg_schema, settings=pg.PostgresSettings(), ) - await duck.connect_to_db(app, settings=duck.DuckDBSettings()) + await duck.connect_to_duckdb(app, settings=duck.DuckDBSettings()) await register_collection_catalog(app) yield - await duck.close_db_connection(app) + await duck.close_duckdb_connection(app) await pg.close_db_connection(app) diff --git a/tests/duck/conftest.py b/tests/duck/conftest.py index b5e4d51f..65ac7e84 100644 --- a/tests/duck/conftest.py +++ b/tests/duck/conftest.py @@ -14,8 +14,8 @@ from tipg.duck import ( # noqa: E402 DuckDBCatalogSettings, DuckDBSettings, - close_db_connection, - connect_to_db, + close_duckdb_connection, + connect_to_duckdb, duck_get_collection_index, ) @@ -101,7 +101,8 @@ def create_duck_app( async def register_collection_catalog(app: FastAPI) -> None: collections = await duck_get_collection_index( - app.state.duckdb_connection, catalog_settings + connection=app.state.duckdb_connection, + settings=catalog_settings, ) app.state.collection_catalog = Catalog( collections={col.id: col for col in collections}, @@ -110,10 +111,10 @@ async def register_collection_catalog(app: FastAPI) -> None: @asynccontextmanager async def lifespan(app: FastAPI): - await connect_to_db(app, settings=settings) + await connect_to_duckdb(app, settings=settings) await register_collection_catalog(app) yield - await close_db_connection(app) + await close_duckdb_connection(app) app = FastAPI(lifespan=lifespan) app.include_router(Endpoints(title="TiPg DuckDB").router) @@ -143,32 +144,33 @@ def mixed_app(database_url, duckdb_settings): from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers from tipg.factory import Endpoints from tipg.pg import ( - DatabaseSettings, + PostgresCatalogSettings, PostgresSettings, + close_db_connection, + connect_to_db, + pg_get_collection_index, ) - from tipg.pg import close_db_connection as pg_close - from tipg.pg import connect_to_db as pg_connect - from tipg.pg import pg_get_collection_index - db_settings = DatabaseSettings(schemas=["public"]) - catalog_settings = DuckDBCatalogSettings(tables=["main.points"]) + postgres_catalog_settings = PostgresCatalogSettings(schemas=["public"]) + duckdb_catalog_settings = DuckDBCatalogSettings(tables=["main.points"]) @asynccontextmanager async def lifespan(app: FastAPI): - await pg_connect( + await connect_to_db( app, - schemas=db_settings.schemas, - tipg_schema=db_settings.tipg_schema, + schemas=postgres_catalog_settings.schemas, + tipg_schema=postgres_catalog_settings.tipg_schema, settings=PostgresSettings(database_url=database_url), ) - await connect_to_db(app, settings=duckdb_settings) + await connect_to_duckdb(app, settings=duckdb_settings) - async with app.state.pool.acquire() as conn: - pg_collections = await pg_get_collection_index( - conn, settings=db_settings, connection_pool=app.state.pool - ) + pg_collections = await pg_get_collection_index( + connection_pool=app.state.pool, + settings=postgres_catalog_settings, + ) duck_collections = await duck_get_collection_index( - app.state.duckdb_connection, catalog_settings + connection=app.state.duckdb_connection, + settings=duckdb_catalog_settings, ) app.state.collection_catalog = Catalog( collections={col.id: col for col in [*pg_collections, *duck_collections]}, @@ -177,8 +179,8 @@ async def lifespan(app: FastAPI): yield + await close_duckdb_connection(app) await close_db_connection(app) - await pg_close(app) app = FastAPI(lifespan=lifespan) app.include_router(Endpoints(title="TiPg mixed").router) diff --git a/tipg/duck/__init__.py b/tipg/duck/__init__.py index 08bea969..966cb31a 100644 --- a/tipg/duck/__init__.py +++ b/tipg/duck/__init__.py @@ -1,14 +1,14 @@ """tipg.duck package: DuckDB integration for TiPg.""" from .collections import DuckCollection, duck_get_collection_index -from .database import close_db_connection, connect_to_db, create_connection +from .database import close_duckdb_connection, connect_to_duckdb, create_connection from .settings import DuckDBCatalogSettings, DuckDBSettings __all__ = [ "DuckCollection", "duck_get_collection_index", - "close_db_connection", - "connect_to_db", + "close_duckdb_connection", + "connect_to_duckdb", "create_connection", "DuckDBCatalogSettings", "DuckDBSettings", diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 55198393..4a95b2ef 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -725,6 +725,7 @@ def _collection_index( # noqa: C901 async def duck_get_collection_index( + *, connection: duckdb.DuckDBPyConnection, settings: DuckDBCatalogSettings | None = None, ) -> list[Collection]: diff --git a/tipg/duck/database.py b/tipg/duck/database.py index 8fd368ec..89f6f15b 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -70,7 +70,7 @@ def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: return conn -async def connect_to_db( +async def connect_to_duckdb( app: FastAPI, *, settings: DuckDBSettings | None = None, @@ -82,6 +82,6 @@ async def connect_to_db( app.state.duckdb_connection = create_connection(settings) -async def close_db_connection(app: FastAPI) -> None: +async def close_duckdb_connection(app: FastAPI) -> None: """Close the DuckDB connection.""" app.state.duckdb_connection.close() From 88b3f222d9394071e4ec6f76af4395c2a4afe33e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 15:20:55 +0100 Subject: [PATCH 14/22] docs: update changelog for the DuckDB backend Co-Authored-By: Claude Opus 5.5 --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 27fc9fc4..3708c80f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,10 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin * perf: use `ST_EstimatedExtent` for spatial extents whenever table statistics exist. * perf: cache `Collection.extent`, `bounds`, `dt_bounds`, `geometry_columns` and `datetime_columns` (`functools.cached_property`) instead of recomputing them on every access * perf: skip SQL statement formatting in `tipg.pg.collections.debug_query` when `DEBUG` logging is disabled +* feat: add DuckDB backend (`tipg.duck`, `tipg[duckdb]` extra) to serve DuckDB tables, views and files (GeoParquet, `ST_Read` formats) as OGC Features and Tiles collections, alone or mixed with PostgreSQL collections in the same catalog + * `connect_to_duckdb` / `close_duckdb_connection` to manage the connection (`app.state.duckdb_connection`) + * `duck_get_collection_index(*, connection, settings=None)` to list the collections + * `DuckDBSettings` / `DuckDBCatalogSettings` (`TIPG_DUCKDB_` env prefix) * change: add `Collection._tag_geometry_literal` and `Collection._reproject_geometry_literal` abstract methods, defining how CQL2/bbox geometry literals are tagged with, or reprojected to, the geometry column CRS for each backend **breaking change** (custom `Collection` subclasses must implement them) * change: move `_quote_ident` from `tipg.pg.collections` to `tipg.collections` * docs: update the `examples/non_app_usage.ipynb` notebook for the new `tipg.pg` API From 8eecf2b1bca670595eb961b6a67c31e3b4b8d451 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 15:55:56 +0100 Subject: [PATCH 15/22] fix: int cast MVT --- tests/duck/conftest.py | 6 ++++++ tests/duck/test_duck.py | 4 ++++ tipg/duck/collections.py | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/duck/conftest.py b/tests/duck/conftest.py index 65ac7e84..85ffdb1d 100644 --- a/tests/duck/conftest.py +++ b/tests/duck/conftest.py @@ -30,6 +30,9 @@ day DATE, tags VARCHAR[], props JSON, + rank TINYINT, + code USMALLINT, + count UINTEGER, geom GEOMETRY ); INSERT INTO points @@ -42,6 +45,9 @@ DATE '2020-01-01' + x::INTEGER, ['a', 'b'], '{"rank": ' || x || '}', + x, + x * 100, + x * 1000000, ST_Point(x, x) FROM range(1, 21) r(x); diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py index ad3ce071..398c1797 100644 --- a/tests/duck/test_duck.py +++ b/tests/duck/test_duck.py @@ -246,6 +246,10 @@ def test_tiles_options(duck_app): assert props["price"] == 2.25 assert props["day"] == "2020-01-02" assert props["tags"] == "[a, b]" + # integer types ST_AsMVT rejects (TINYINT, USMALLINT, UINTEGER) are cast + assert props["rank"] == 1 + assert props["code"] == 100 + assert props["count"] == 1000000 response = duck_app.get( url, params={"sortby": "-id", "limit": 3, "properties": "id"} diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 4a95b2ef..59cc8dbf 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -21,18 +21,16 @@ features_settings = FeaturesSettings() # type: ignore [call-arg] # Column types (as normalized in the catalog) ST_AsMVT can encode as-is -_MVT_NATIVE_TYPES = frozenset( - { - "smallint", - "integer", - "bigint", - "real", - "float8", - "boolean", - "text", - "json", - } -) +_MVT_NATIVE_TYPES = frozenset({"real", "float8", "boolean", "text", "json"}) + +# Integer types (as normalized in the catalog) → DuckDB type ST_AsMVT accepts. +# The normalized name may come from a type ST_AsMVT rejects (e.g. ``UINTEGER`` +# is normalized to ``bigint``, ``TINYINT`` to ``smallint``), so always cast. +_MVT_INTEGER_CASTS = { + "smallint": "INTEGER", + "integer": "INTEGER", + "bigint": "BIGINT", +} def _quote_literal(value: str) -> str: @@ -88,6 +86,8 @@ def _select_mvt_property_columns(self, properties: list[str] | None) -> list[str q = _quote_ident(name) if col is None or col.type in _MVT_NATIVE_TYPES: cols.append(q) + elif cast := _MVT_INTEGER_CASTS.get(col.type): + cols.append(f"CAST({q} AS {cast}) AS {q}") elif col.type == "numeric": cols.append(f"CAST({q} AS DOUBLE) AS {q}") else: From 26f73f5ecadf619f10255da4a5ca3f8195fac1ba Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 19:49:20 +0100 Subject: [PATCH 16/22] perf: avoid full scans --- docs/src/advanced/duckdb.md | 5 +- examples/main.py | 113 ++++++++++++++++++++ tests/duck/test_duck.py | 38 ++++--- tests/duck/test_geoparquet.py | 95 +++++++++++++++++ tipg/duck/collections.py | 190 ++++++++++++++++++++++------------ tipg/duck/settings.py | 16 ++- 6 files changed, 373 insertions(+), 84 deletions(-) create mode 100644 examples/main.py create mode 100644 tests/duck/test_geoparquet.py diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index ffce5501..2015afca 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -25,8 +25,7 @@ python -m pip install "tipg[duckdb]" | `tables` | `TIPG_DUCKDB_TABLES` | | Only expose those tables (`schema.table`) | | `exclude_tables` | `TIPG_DUCKDB_EXCLUDE_TABLES` | | Tables (`schema.table`) to exclude | | `only_spatial_tables` | `TIPG_DUCKDB_ONLY_SPATIAL_TABLES` | `True` | Only expose tables/views with a geometry column | -| `spatial_extent` | `TIPG_DUCKDB_SPATIAL_EXTENT` | `True` | Compute the spatial extent of each collection | -| `datetime_extent` | `TIPG_DUCKDB_DATETIME_EXTENT` | `True` | Compute the temporal extent of each collection | +| `spatial_extent` | `TIPG_DUCKDB_SPATIAL_EXTENT` | `True` | Read the spatial extent and geometry type of parquet sources from their GeoParquet metadata (file footers only). For a glob, every matched file's footer is read on each catalog refresh: disable it for sources matching many remote files | Table level configuration (`TIPG_TABLE_CONFIG__...`, fallback key names, column sorting) is shared with the PostgreSQL backend (see [Configuration](../user_guide/configuration.md)). @@ -158,7 +157,7 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) ### Differences with the PostgreSQL backend - **CRS**: the SRID of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. -- **Spatial extent**: computed with a full scan of the geometry column (DuckDB has no equivalent of PostGIS' `ST_EstimatedExtent`). Set `TIPG_DUCKDB_SPATIAL_EXTENT=FALSE` for large datasets. +- **Extents and geometry type**: tables and views are never scanned to build the catalog. The spatial extent and geometry type of parquet `sources` are read from their [GeoParquet](https://geoparquet.org) metadata (`bbox` and `geometry_types`, in the file footers). Other collections have no spatial extent and a generic `Geometry` type, and no collection has a temporal extent. - **Vector tiles**: `ST_AsMVT` only supports boolean, integer, float and string properties; `DECIMAL` values are cast to `DOUBLE` and other types (dates, timestamps, lists, ...) to `VARCHAR`. - **Functions**: SQL function layers are not supported. - **WKT output** (CSV, (nd)json): DuckDB's `ST_AsText` formatting (e.g `SRID=4326;POINT (1 2)`). diff --git a/examples/main.py b/examples/main.py new file mode 100644 index 00000000..1624f396 --- /dev/null +++ b/examples/main.py @@ -0,0 +1,113 @@ +"""tipg app with DuckDB backend.""" + +import datetime +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from starlette.middleware.cors import CORSMiddleware +from starlette_cramjam.middleware import CompressionMiddleware + +from tipg import __version__ as tipg_version +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.extensions.viewer import viewerExtension +from tipg.factory import Endpoints +from tipg.middleware import CacheControlMiddleware +from tipg.openapi import _update_openapi +from tipg.settings import APISettings + +settings = APISettings() + +duckdb_catalog_settings = DuckDBCatalogSettings() + + +async def register_collection_catalog( + app: FastAPI, + catalog_settings: DuckDBCatalogSettings, +) -> None: + """Register Table catalog.""" + duck_collections = await duck_get_collection_index( + connection=app.state.duckdb_connection, + settings=catalog_settings, + ) + + app.state.collection_catalog = Catalog( + collections={col.id: col for col in [*duck_collections]}, + last_updated=datetime.datetime.now(), + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + # Create Connection Pool + await connect_to_duckdb(app) + + # Register Collection Catalog + await register_collection_catalog( + app, + catalog_settings=duckdb_catalog_settings, + ) + + yield + + # Close the Connection Pool + await close_duckdb_connection(app) + + +app = FastAPI( + title=settings.name, + version=tipg_version, + openapi_url="/api", + docs_url="/api.html", + lifespan=lifespan, + root_path=settings.root_path, +) + +# Fix OpenAPI response header for OGC Common compatibility +_update_openapi(app) + +ogc_api = Endpoints( + title=settings.name, + with_tiles_viewer=settings.add_tiles_viewer, + extensions=[viewerExtension()], +) +app.include_router(ogc_api.router) + + +# Set all CORS enabled origins +if settings.cors_origins: + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["GET"], + allow_headers=["*"], + ) + +app.add_middleware( + CacheControlMiddleware, + cachecontrol=settings.cachecontrol, + exclude_path={r"^/healthz$", r"^/metrics$"}, +) +app.add_middleware(CompressionMiddleware, compression_level=6) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) + + +@app.get( + "/healthz", + description="Health Check.", + summary="Health Check.", + operation_id="healthCheck", + tags=["Health Check"], +) +def ping(): + """Health check.""" + return {"ping": "pong!"} diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py index 398c1797..7293d654 100644 --- a/tests/duck/test_duck.py +++ b/tests/duck/test_duck.py @@ -35,19 +35,21 @@ def test_collections(duck_app): def test_collection_metadata(duck_app): """Check extents, CRS and queryables.""" + # tables are not scanned: no extents, generic geometry type body = duck_app.get("/collections/main.points").json() - assert body["extent"]["spatial"]["bbox"] == [[1.0, 1.0, 20.0, 20.0]] - intervals = body["extent"]["temporal"]["interval"] - # `day` (DATE) is the default datetime column (columns are sorted by name) - assert intervals[0] == ["2020-01-01T01:00:00+00:00", "2020-01-21T00:00:00+00:00"] - assert ["2020-01-01T01:00:00+00:00", "2020-01-01T20:00:00+00:00"] in intervals - - # projected table: extent is reprojected to WGS84 - body = duck_app.get("/collections/main.utm").json() - minx, miny, maxx, maxy = body["extent"]["spatial"]["bbox"][0] - assert -94 < minx < maxx < -92 - assert 44 < miny < maxy < 47 - utm = duck_app.app.state.collection_catalog["collections"]["main.utm"] + assert body["extent"]["spatial"]["bbox"] == [[-180.0, -90.0, 180.0, 90.0]] + assert "temporal" not in body["extent"] + catalog = duck_app.app.state.collection_catalog["collections"] + assert catalog["main.points"].geometry_column.bounds == [-180, -90, 180, 90] + assert catalog["main.points"].geometry_column.geometry_type == "Geometry" + assert catalog["main.points"].datetime_column.mindt is None + + # parquet source: extent and geometry type from the GeoParquet metadata + body = duck_app.get("/collections/main.landmarks").json() + assert body["extent"]["spatial"]["bbox"] == [[1.0, 1.0, 5.0, 5.0]] + assert catalog["main.landmarks"].geometry_column.geometry_type == "Point" + + utm = catalog["main.utm"] assert utm.geometry_column.srid == 26915 assert utm.crs == "http://www.opengis.net/def/crs/EPSG/0/26915" @@ -56,7 +58,7 @@ def test_collection_metadata(duck_app): body = duck_app.get("/collections/main.points/queryables").json() props = body["properties"] - assert props["geom"] == {"$ref": "https://geojson.org/schema/Point.json"} + assert props["geom"] == {"$ref": "https://geojson.org/schema/Geometry.json"} assert props["id"]["type"] == "number" assert props["price"]["type"] == "number" assert props["name"]["type"] == "string" @@ -68,8 +70,14 @@ def test_collections_filters(duck_app): """Filter collections by bbox / datetime (uses collection extents).""" body = duck_app.get("/collections", params={"bbox": "0,0,5,5"}).json() ids = {c["id"] for c in body["collections"]} - assert {"main.points", "main.landmarks"} <= ids - assert "main.utm" not in ids + # tables have no extent (world bbox), so they always match + assert {"main.points", "main.landmarks", "main.utm"} <= ids + + # landmarks extent (GeoParquet metadata) is [1, 1, 5, 5] + body = duck_app.get("/collections", params={"bbox": "10,10,20,20"}).json() + ids = {c["id"] for c in body["collections"]} + assert "main.landmarks" not in ids + assert "main.points" in ids body = duck_app.get( "/collections", params={"datetime": "2019-01-01T00:00:00Z/2019-12-31T00:00:00Z"} diff --git a/tests/duck/test_geoparquet.py b/tests/duck/test_geoparquet.py new file mode 100644 index 00000000..4a52c696 --- /dev/null +++ b/tests/duck/test_geoparquet.py @@ -0,0 +1,95 @@ +"""Test GeoParquet metadata (bbox, geometry types) used by the DuckDB catalog.""" + +import pytest + +from tipg.duck import DuckDBCatalogSettings, DuckDBSettings +from tipg.duck.collections import ( + _collection_index, + _geoparquet_metadata, + _parquet_view_source, +) +from tipg.duck.database import create_connection + +duckdb = pytest.importorskip("duckdb") + + +@pytest.fixture(scope="module") +def parquet_dir(tmp_path_factory): + """Write GeoParquet files (DuckDB writes the `geo` metadata).""" + tmp = tmp_path_factory.mktemp("geoparquet") + conn = duckdb.connect() + conn.execute("LOAD spatial") + conn.execute( + f"COPY (SELECT 1 AS id, ST_Point(1, 2) AS geom) TO '{tmp}/part-a.parquet'" + ) + conn.execute( + f"COPY (SELECT 2 AS id, ST_Buffer(ST_Point(10, 20), 1) AS geom) TO '{tmp}/part-b.parquet'" + ) + conn.execute(f"COPY (SELECT 1 AS id, 'a' AS name) TO '{tmp}/nongeo.parquet'") + conn.close() + return tmp + + +def test_parquet_view_source(): + """Only plain `SELECT * FROM read_parquet(...)` views are recognized.""" + assert ( + _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('s3://b/it''s/*.parquet');" + ) + == "s3://b/it's/*.parquet" + ) + assert _parquet_view_source(None) is None + assert ( + _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('x.parquet') WHERE id > 1;" + ) + is None + ) + assert ( + _parquet_view_source("CREATE VIEW a AS SELECT * FROM ST_Read('x.shp');") is None + ) + + +def test_geoparquet_metadata(parquet_dir): + """Read and merge bbox/geometry types from the file footers.""" + conn = duckdb.connect() + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-a.parquet") + assert meta == {"geom": {"bbox": [1.0, 2.0, 1.0, 2.0], "geometry_types": {"Point"}}} + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-*.parquet") + assert meta["geom"]["bbox"] == [1.0, 2.0, 11.0, 21.0] + assert meta["geom"]["geometry_types"] == {"Point", "Polygon"} + + # no `geo` metadata + assert _geoparquet_metadata(conn, f"{parquet_dir}/nongeo.parquet") == {} + # mixed files: some files have no `geo` metadata + assert _geoparquet_metadata(conn, f"{parquet_dir}/*.parquet") == {} + # errors fall back to an empty dict + assert _geoparquet_metadata(conn, f"{parquet_dir}/missing.parquet") == {} + + +def test_catalog_uses_geoparquet_metadata(parquet_dir): + """Collection extent and geometry type come from the GeoParquet metadata.""" + conn = create_connection( + DuckDBSettings( + sources={ + "single": f"{parquet_dir}/part-a.parquet", + "multi": f"{parquet_dir}/part-*.parquet", + } + ) + ) + try: + collections = { + c.id: c for c in _collection_index(conn, DuckDBCatalogSettings()) + } + finally: + conn.close() + + single = collections["main.single"].geometry_column + assert single.bounds == [1.0, 2.0, 1.0, 2.0] + assert single.geometry_type == "Point" + + multi = collections["main.multi"].geometry_column + assert multi.bounds == [1.0, 2.0, 11.0, 21.0] + assert multi.geometry_type == "Geometry" diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 59cc8dbf..f4429525 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -1,6 +1,5 @@ """tipg DuckDB collection models.""" -import datetime import re from typing import Any @@ -514,75 +513,135 @@ def _geometry_srid(data_type: str) -> int: return 4326 -def _iso_datetime(value: Any) -> str | None: - """ISO format a date/datetime extent (naive values are assumed to be UTC).""" - if value is None: - return None +_PARQUET_VIEW = re.compile( + r"^CREATE VIEW .+? AS SELECT \* FROM read_parquet\('(?P(?:[^']|'')*)'\);?$", + re.DOTALL | re.IGNORECASE, +) - if not isinstance(value, datetime.datetime): - value = datetime.datetime.combine(value, datetime.time()) - if value.tzinfo is None: - value = value.replace(tzinfo=datetime.timezone.utc) +def _parquet_view_source(view_sql: str | None) -> str | None: + """Path (or glob) of a view defined as ``SELECT * FROM read_parquet('')``.""" + if view_sql and (m := _PARQUET_VIEW.match(view_sql.strip())): + return m.group("path").replace("''", "'") - return value.isoformat() + return None -def _table_metadata( # noqa: C901 +def _parse_geo_metadata(geo: str) -> dict[str, dict[str, Any]]: + """Bbox and geometry types per column, from a GeoParquet ``geo`` metadata value.""" + columns: dict[str, dict[str, Any]] = {} + for name, meta in orjson.loads(geo).get("columns", {}).items(): + bbox = meta.get("bbox") + if not bbox or len(bbox) not in (4, 6): + continue + + # 3D bbox: [xmin, ymin, zmin, xmax, ymax, zmax] + if len(bbox) == 6: + bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] + + types = meta.get("geometry_types") + columns[name] = { + "bbox": [float(v) for v in bbox], + "geometry_types": set(types) if types else None, + } + + return columns + + +def _geoparquet_metadata( cursor: duckdb.DuckDBPyConnection, - qualified_name: str, - columns: list[dict[str, Any]], - settings: DuckDBCatalogSettings, + path: str, +) -> dict[str, dict[str, Any]]: + """Bbox and geometry types of geometry columns, from GeoParquet metadata. + + Only the file footers are read. For multiple files (glob), bboxes and + geometry types are merged, and a column is only returned when every file + has a bbox for it. Geometry types are ``None`` when unknown. + """ + try: + nfiles = cursor.execute("SELECT count(*) FROM glob($1)", [path]).fetchone()[0] # type: ignore [index] + rows = cursor.execute( + """ + SELECT max(decode(value)) FILTER (WHERE decode(key) = 'geo') + FROM parquet_kv_metadata($1) + GROUP BY file_name + """, + [path], + ).fetchall() + except duckdb.Error as e: + logger.debug(f"Could not read GeoParquet metadata of {path}: {e}") + return {} + + # files without any key/value metadata are not listed by `parquet_kv_metadata` + if not rows or len(rows) != nfiles or any(geo is None for (geo,) in rows): + return {} + + merged: dict[str, dict[str, Any]] | None = None + for (geo,) in rows: + file_columns = _parse_geo_metadata(geo) + if merged is None: + merged = file_columns + continue + + for name in list(merged): + if (other := file_columns.get(name)) is None: + del merged[name] + continue + + col = merged[name] + col["bbox"] = [ + min(col["bbox"][0], other["bbox"][0]), + min(col["bbox"][1], other["bbox"][1]), + max(col["bbox"][2], other["bbox"][2]), + max(col["bbox"][3], other["bbox"][3]), + ] + if col["geometry_types"] is None or other["geometry_types"] is None: + col["geometry_types"] = None + else: + col["geometry_types"] |= other["geometry_types"] + + return merged or {} + + +def _apply_geo_metadata( + cursor: duckdb.DuckDBPyConnection, + column: dict[str, Any], + meta: dict[str, Any], ) -> None: - """Add geometry type, bounds and datetime extents to the columns (in place).""" - exprs: list[str] = [] - for c in columns: - q = _quote_ident(c["name"]) - if c["type"] == "geometry": - c["geometry_type"] = "Geometry" - if settings.spatial_extent: - extent = f"ST_Extent_Agg({q})" - if c["srid"] != 4326: - extent = ( - f"ST_Transform({extent}, '{_crs(c['srid'])}', '{_crs(4326)}')" - ) - exprs.extend( - [ - f"ST_XMin({extent})", - f"ST_YMin({extent})", - f"ST_XMax({extent})", - f"ST_YMax({extent})", - f"CAST(min(ST_GeometryType({q})) AS VARCHAR)", - f"CAST(max(ST_GeometryType({q})) AS VARCHAR)", - ] - ) + """Set a geometry column bounds (in EPSG:4326) and type from GeoParquet metadata.""" + xmin, ymin, xmax, ymax = meta["bbox"] + if column["srid"] != 4326: + env = f"ST_Transform(ST_MakeEnvelope($1, $2, $3, $4), '{_crs(column['srid'])}', '{_crs(4326)}')" + xmin, ymin, xmax, ymax = cursor.execute( + f"SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e) FROM (SELECT {env} AS e)", + [xmin, ymin, xmax, ymax], + ).fetchone() # type: ignore [misc] - elif ( - c["type"] in ("timestamp", "timestamptz", "date") - and settings.datetime_extent - ): - exprs.extend([f"min({q})", f"max({q})"]) + column["bounds"] = [xmin, ymin, xmax, ymax] - if not exprs: - return + # e.g. "Polygon Z" -> "POLYGON" + if types := meta["geometry_types"]: + base_types = {t.split()[0].upper() for t in types} + if len(base_types) == 1: + column["geometry_type"] = _GEOMETRY_TYPES.get(base_types.pop(), "Geometry") - values = iter( - cursor.execute(f"SELECT {', '.join(exprs)} FROM {qualified_name}").fetchone() # type: ignore [arg-type] - ) - for c in columns: - if c["type"] == "geometry" and settings.spatial_extent: - xmin, ymin, xmax, ymax, gmin, gmax = (next(values) for _ in range(6)) - if xmin is not None: - c["bounds"] = [xmin, ymin, xmax, ymax] - if gmin is not None and gmin == gmax: - c["geometry_type"] = _GEOMETRY_TYPES.get(gmin.upper(), "Geometry") - elif ( - c["type"] in ("timestamp", "timestamptz", "date") - and settings.datetime_extent - ): - c["mindt"] = _iso_datetime(next(values)) - c["maxdt"] = _iso_datetime(next(values)) +def _geometry_metadata( + cursor: duckdb.DuckDBPyConnection, + columns: list[dict[str, Any]], + geo_metadata: dict[str, dict[str, Any]], +) -> None: + """Add geometry type and bounds to the geometry columns (in place). + + Rows are never scanned: bounds and type only come from ``geo_metadata`` + (GeoParquet file metadata). Other geometry columns have no bounds and a + generic ``Geometry`` type. + """ + for c in columns: + if c["type"] == "geometry": + c["geometry_type"] = "Geometry" + if meta := geo_metadata.get(c["name"]): + _apply_geo_metadata(cursor, c, meta) def _collection_index( # noqa: C901 @@ -594,11 +653,11 @@ def _collection_index( # noqa: C901 try: relations = cursor.execute( """ - SELECT database_name, schema_name, table_name, 'Table', comment + SELECT database_name, schema_name, table_name, 'Table', comment, NULL FROM duckdb_tables() WHERE NOT internal AND NOT temporary UNION ALL - SELECT database_name, schema_name, view_name, 'View', comment + SELECT database_name, schema_name, view_name, 'View', comment, sql FROM duckdb_views() WHERE NOT internal AND NOT temporary ORDER BY 1, 2, 3 @@ -644,7 +703,7 @@ def _collection_index( # noqa: C901 fallback_key_names = table_settings.fallback_key_names collections: dict[str, Collection] = {} - for db, schema, table, entity, comment in relations: + for db, schema, table, entity, comment, view_sql in relations: if schema not in settings.schemas: continue @@ -667,8 +726,11 @@ def _collection_index( # noqa: C901 ): continue - qualified_name = ".".join(_quote_ident(p) for p in (db, schema, table)) - _table_metadata(cursor, qualified_name, columns, settings) + geo_metadata = {} + if settings.spatial_extent and (path := _parquet_view_source(view_sql)): + geo_metadata = _geoparquet_metadata(cursor, path) + + _geometry_metadata(cursor, columns, geo_metadata) table_conf = table_confs.get(f"{schema}_{table}", TableConfig()) diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py index 1fa610e1..5732b8fb 100644 --- a/tipg/duck/settings.py +++ b/tipg/duck/settings.py @@ -32,12 +32,24 @@ class DuckDBSettings(BaseSettings): class DuckDBCatalogSettings(BaseSettings): - """TiPg DuckDB catalog settings.""" + """TiPg DuckDB catalog settings. + + Attributes: + spatial_extent: read the spatial extent and geometry type of parquet + sources from their GeoParquet metadata (`bbox` and `geometry_types`). + Only the file footers are read, rows are never scanned. For a glob, + the footer of every matched file is read (one request per file for + remote files), on startup and on each catalog refresh: disable it + for sources matching many (remote) files. When disabled, or when + the metadata is missing, collections have no spatial extent + (world bbox) and a generic `Geometry` type. Tables and other views + never have a spatial extent. + + """ schemas: list[str] = ["main"] tables: list[str] | None = None exclude_tables: list[str] | None = None - datetime_extent: bool = True spatial_extent: bool = True only_spatial_tables: bool = True From 2861b041618b3ef1399a40938873af1513aa66f9 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 10 Nov 2026 20:32:21 +0100 Subject: [PATCH 17/22] perf: improve performance for items and tiles --- docs/src/advanced/duckdb.md | 5 + examples/main.py | 7 +- tests/duck/test_duck.py | 21 +- tests/duck/test_geoparquet.py | 271 ++++++++++++++++++++++++- tipg/duck/collections.py | 368 +++++++++++++++++++++++++++------- tipg/duck/database.py | 44 +++- tipg/duck/settings.py | 4 +- 7 files changed, 627 insertions(+), 93 deletions(-) diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index 2015afca..afad90fc 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -158,6 +158,11 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) - **CRS**: the SRID of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. - **Extents and geometry type**: tables and views are never scanned to build the catalog. The spatial extent and geometry type of parquet `sources` are read from their [GeoParquet](https://geoparquet.org) metadata (`bbox` and `geometry_types`, in the file footers). Other collections have no spatial extent and a generic `Geometry` type, and no collection has a temporal extent. +- **Items order**: without `sortby`, features are returned in their insertion order (no default sort on the primary key). +- **Feature ids**: single parquet file `sources` are read with `read_parquet(..., file_row_number = true)`, and the added `file_row_number` column (row position in the file, starting at 0) is used as id when the collection has no other id column (`TIPG_TABLE_CONFIG__..._pk` or fallback key names). Looking up a feature by id (`/items/{itemId}`) then only reads one row group. Other collections without primary key use the feature position in the response as `id`, and don't support `/items/{itemId}`. +- **Spatial filters**: DuckDB has no spatial index on parquet files. When a GeoParquet (1.1+) source declares a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column, spatial filters (`bbox`, tile envelope, and cql2 spatial predicates in the top-level `AND` of the filter) also filter the covering columns, so DuckDB can skip the row groups outside of the filter using their statistics. Without covering, spatial filters scan the whole geometry column. +- **Parquet metadata cache**: parquet files metadata (footers) are cached in memory (`parquet_metadata_cache`), so they are only fetched once per file. +- **`numberMatched`**: only returned for requests without filter (DuckDB answers `COUNT(*)` from metadata), counting the features matching a filter would need a full scan. - **Vector tiles**: `ST_AsMVT` only supports boolean, integer, float and string properties; `DECIMAL` values are cast to `DOUBLE` and other types (dates, timestamps, lists, ...) to `VARCHAR`. - **Functions**: SQL function layers are not supported. - **WKT output** (CSV, (nd)json): DuckDB's `ST_AsText` formatting (e.g `SRID=4326;POINT (1 2)`). diff --git a/examples/main.py b/examples/main.py index 1624f396..869e7494 100644 --- a/examples/main.py +++ b/examples/main.py @@ -1,4 +1,9 @@ -"""tipg app with DuckDB backend.""" +"""tipg app with DuckDB backend. + +export TIPG_DUCKDB_SOURCES='{"test":"https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' +uv run --group server uvicorn main:app + +""" import datetime from contextlib import asynccontextmanager diff --git a/tests/duck/test_duck.py b/tests/duck/test_duck.py index 7293d654..d9e71ed1 100644 --- a/tests/duck/test_duck.py +++ b/tests/duck/test_duck.py @@ -139,7 +139,26 @@ def test_items_filters(duck_app): url, params={"datetime": "2020-01-01T03:00:00Z/..", "datetime-column": "dt"}, ).json() - assert body["numberMatched"] == 18 + # filtered requests don't count the matched features (full scan) + assert body.get("numberMatched") is None + assert [f["id"] for f in body["features"]] == list(range(3, 13)) + assert "next" in {link["rel"] for link in body["links"]} + + body = duck_app.get( + url, + params={ + "datetime": "2020-01-01T03:00:00Z/..", + "datetime-column": "dt", + "offset": 10, + }, + ).json() + assert [f["id"] for f in body["features"]] == list(range(13, 21)) + assert "next" not in {link["rel"] for link in body["links"]} + + # exactly `limit` features left: no next link + body = duck_app.get(url, params={"ids": "2,4", "limit": 2}).json() + assert len(body["features"]) == 2 + assert "next" not in {link["rel"] for link in body["links"]} body = duck_app.get(url, params={"name": "point 3"}).json() assert [f["id"] for f in body["features"]] == [3] diff --git a/tests/duck/test_geoparquet.py b/tests/duck/test_geoparquet.py index 4a52c696..944cac85 100644 --- a/tests/duck/test_geoparquet.py +++ b/tests/duck/test_geoparquet.py @@ -1,11 +1,17 @@ """Test GeoParquet metadata (bbox, geometry types) used by the DuckDB catalog.""" +import asyncio +import json + +import morecantile import pytest +from cql2 import Expr from tipg.duck import DuckDBCatalogSettings, DuckDBSettings from tipg.duck.collections import ( _collection_index, _geoparquet_metadata, + _literal_bounds, _parquet_view_source, ) from tipg.duck.database import create_connection @@ -26,18 +32,66 @@ def parquet_dir(tmp_path_factory): f"COPY (SELECT 2 AS id, ST_Buffer(ST_Point(10, 20), 1) AS geom) TO '{tmp}/part-b.parquet'" ) conn.execute(f"COPY (SELECT 1 AS id, 'a' AS name) TO '{tmp}/nongeo.parquet'") + # no id column, `value` not in insertion order + conn.execute( + f"COPY (SELECT 'f' || x AS name, 100 - x AS value, ST_Point(x, x) AS geom " + f"FROM range(1, 26) r(x)) TO '{tmp}/noid.parquet'" + ) + conn.execute( + f"COPY (SELECT 42 AS file_row_number, ST_Point(0, 0) AS geom) TO '{tmp}/frn.parquet'" + ) + + # GeoParquet 1.1 with a bbox covering column (5 row groups, points on a + # diagonal from 0 to 99.99): DuckDB doesn't write coverings, so the + # geometry is written as WKB with our own `geo` metadata + (tmp / "covering").mkdir() + geo = json.dumps( + { + "version": "1.1.0", + "primary_column": "geom", + "columns": { + "geom": { + "encoding": "WKB", + "geometry_types": ["Point"], + "bbox": [0, 0, 99.99, 99.99], + "covering": { + "bbox": { + "xmin": ["bbox", "xmin"], + "ymin": ["bbox", "ymin"], + "xmax": ["bbox", "xmax"], + "ymax": ["bbox", "ymax"], + } + }, + } + }, + } + ) + conn.execute( + f""" + COPY ( + SELECT + x AS fid, + ST_AsWKB(ST_Point(x / 100, x / 100)) AS geom, + {{'xmin': x / 100, 'ymin': x / 100, 'xmax': x / 100, 'ymax': x / 100}} AS bbox + FROM range(0, 10000) r(x) + ) TO '{tmp}/covering/points.parquet' + (FORMAT parquet, ROW_GROUP_SIZE 2048, KV_METADATA {{geo: '{geo}'}}) + """ + ) + conn.close() return tmp def test_parquet_view_source(): """Only plain `SELECT * FROM read_parquet(...)` views are recognized.""" - assert ( - _parquet_view_source( - "CREATE VIEW a AS SELECT * FROM read_parquet('s3://b/it''s/*.parquet');" - ) - == "s3://b/it's/*.parquet" - ) + assert _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('s3://b/it''s/*.parquet');" + ) == ("s3://b/it's/*.parquet", False) + # as stored by DuckDB for `read_parquet('x.parquet', file_row_number = true)` + assert _parquet_view_source( + "CREATE VIEW a AS SELECT * FROM read_parquet('x.parquet', (file_row_number = true));" + ) == ("x.parquet", True) assert _parquet_view_source(None) is None assert ( _parquet_view_source( @@ -55,7 +109,21 @@ def test_geoparquet_metadata(parquet_dir): conn = duckdb.connect() meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-a.parquet") - assert meta == {"geom": {"bbox": [1.0, 2.0, 1.0, 2.0], "geometry_types": {"Point"}}} + assert meta == { + "geom": { + "bbox": [1.0, 2.0, 1.0, 2.0], + "geometry_types": {"Point"}, + "covering": None, + } + } + + meta = _geoparquet_metadata(conn, f"{parquet_dir}/covering/points.parquet") + assert meta["geom"]["covering"] == { + "xmin": '"bbox"."xmin"', + "ymin": '"bbox"."ymin"', + "xmax": '"bbox"."xmax"', + "ymax": '"bbox"."ymax"', + } meta = _geoparquet_metadata(conn, f"{parquet_dir}/part-*.parquet") assert meta["geom"]["bbox"] == [1.0, 2.0, 11.0, 21.0] @@ -93,3 +161,192 @@ def test_catalog_uses_geoparquet_metadata(parquet_dir): multi = collections["main.multi"].geometry_column assert multi.bounds == [1.0, 2.0, 11.0, 21.0] assert multi.geometry_type == "Geometry" + + +def test_features_without_id_column(parquet_dir): + """No default sort, ids are row positions for collections without primary key.""" + conn = create_connection( + # a glob: no `file_row_number` id (row numbers are not unique across files) + DuckDBSettings(sources={"noid": f"{parquet_dir}/noid*.parquet"}) + ) + try: + (collection,) = _collection_index(conn, DuckDBCatalogSettings()) + assert collection.id_column is None + # no ORDER BY without `sortby` (would be a full scan) + assert collection._sortby(None) == "" + + items = asyncio.run(collection.features(limit=10, offset=10)) + assert items["matched"] == 25 + assert [f["id"] for f in items["items"]] == list(range(11, 21)) + # insertion order + assert [f["properties"]["name"] for f in items["items"]] == [ + f"f{x}" for x in range(11, 21) + ] + + # explicit sort: ids are positions in the sorted features + items = asyncio.run(collection.features(sortby="value", limit=5)) + assert [f["id"] for f in items["items"]] == [1, 2, 3, 4, 5] + assert [f["properties"]["value"] for f in items["items"]] == [ + 75, + 76, + 77, + 78, + 79, + ] + finally: + conn.close() + + +def test_file_row_number_id(parquet_dir): + """Single parquet file sources use the row position in the file as id.""" + conn = create_connection( + DuckDBSettings( + sources={ + "noid": f"{parquet_dir}/noid.parquet", + # already has a `file_row_number` column + "frn": f"{parquet_dir}/frn.parquet", + } + ) + ) + try: + collections = { + c.id: c for c in _collection_index(conn, DuckDBCatalogSettings()) + } + col = collections["main.noid"] + assert col.id_column.name == "file_row_number" + + # ids are stable: same id for a feature with or without filter/sort + items = asyncio.run(col.features(limit=3)) + assert [f["id"] for f in items["items"]] == [0, 1, 2] + assert [f["properties"]["name"] for f in items["items"]] == ["f1", "f2", "f3"] + + items = asyncio.run(col.features(cql_filter=Expr("value < 80"), limit=2)) + assert [f["id"] for f in items["items"]] == [20, 21] + assert [f["properties"]["name"] for f in items["items"]] == ["f21", "f22"] + + items = asyncio.run(col.features(sortby="-name", limit=1)) + assert [f["id"] for f in items["items"]] == [8] # "f9" + + # item lookup + items = asyncio.run(col.features(ids_filter=["20"])) + assert [f["properties"]["name"] for f in items["items"]] == ["f21"] + + # existing `file_row_number` column: plain view, regular column + col = collections["main.frn"] + assert col.get_column("file_row_number") + items = asyncio.run(col.features()) + assert items["items"][0]["properties"]["file_row_number"] == 42 + finally: + conn.close() + + +def test_literal_bounds(): + """Bounds of cql2-json geometry literals.""" + point = {"type": "Point", "coordinates": [1, 2]} + polygon = { + "type": "Polygon", + "coordinates": [[[0, 0], [2, 0], [2, 3], [0, 3], [0, 0]]], + } + assert _literal_bounds(point, 4326) == [1.0, 2.0, 1.0, 2.0] + assert _literal_bounds(polygon, 4326) == [0.0, 0.0, 2.0, 3.0] + assert _literal_bounds({"bbox": [0, 1, 2, 3]}, 4326) == [0.0, 1.0, 2.0, 3.0] + assert _literal_bounds( + {"type": "GeometryCollection", "geometries": [point, polygon]}, 4326 + ) == [0.0, 0.0, 2.0, 3.0] + # tagged with the column CRS: coordinates are in the column CRS + tagged = {"op": "st_setcrs", "args": [polygon, "EPSG:26915"]} + assert _literal_bounds(tagged, 26915) == [0.0, 0.0, 2.0, 3.0] + # untagged literal for a projected column, or reprojected literal: ignored + assert _literal_bounds(polygon, 26915) is None + transformed = {"op": "st_transform", "args": [polygon, "EPSG:4326", "EPSG:26915"]} + assert _literal_bounds(transformed, 26915) is None + assert _literal_bounds({"property": "geom"}, 4326) is None + + +@pytest.fixture +def covering_collection(parquet_dir): + """Collection of a GeoParquet file with a bbox covering column.""" + conn = create_connection( + DuckDBSettings(sources={"cov": f"{parquet_dir}/covering/points.parquet"}) + ) + (collection,) = _collection_index(conn, DuckDBCatalogSettings()) + yield collection + conn.close() + + +def _row_groups_scanned(collection, where_clause: str) -> str: + """`Row Groups Scanned: x / y` from the query profile.""" + plan = collection.connection.execute( + f"EXPLAIN ANALYZE SELECT count(*) {collection._from()} {where_clause}" + ).fetchall()[0][1] + (line,) = [line for line in plan.splitlines() if "Row Groups Scanned" in line] + return line.strip("│ ").split(":")[1].strip() + + +def test_bbox_covering_filter(covering_collection): + """Spatial filters also filter the bbox covering columns (row group pruning).""" + col = covering_collection + assert col.bbox_coverings == { + "geom": { + "xmin": '"bbox"."xmin"', + "ymin": '"bbox"."ymin"', + "xmax": '"bbox"."xmax"', + "ymax": '"bbox"."ymax"', + } + } + assert col.geometry_column.geometry_type == "Point" + + # no spatial filter + assert col._where_clause(None) == "WHERE TRUE" + where = col.cql_where(cql=Expr("fid > 10")) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + + # bbox filter + where = col.cql_where(bbox=[12, 12, 15, 15]) + clause = col._where_clause(where) + assert clause.startswith( + 'WHERE "bbox"."xmin" <= 15.0 AND "bbox"."xmax" >= 12.0 ' + 'AND "bbox"."ymin" <= 15.0 AND "bbox"."ymax" >= 12.0 AND (' + ) + assert _row_groups_scanned(col, clause) == "1 / 5" + assert _row_groups_scanned(col, f"WHERE {where.to_sql()}") == "5 / 5" + + items = asyncio.run(col.features(bbox_filter=[12, 12, 15, 15], limit=1000)) + assert len(items["items"]) == 301 + assert items["matched"] is None + + # cql2 spatial filter, combined with other filters + where = col.cql_where( + cql=Expr( + "fid > 10 AND S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))" + ) + ) + assert _row_groups_scanned(col, col._where_clause(where)) == "1 / 5" + + # spatial filter under OR/NOT: the covering filter can't be applied + where = col.cql_where( + cql=Expr( + "fid = 1 OR S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))" + ) + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + where = col.cql_where( + cql=Expr("NOT S_INTERSECTS(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))") + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + where = col.cql_where( + cql=Expr("S_DISJOINT(geom, POLYGON((12 12, 15 12, 15 15, 12 15, 12 12)))") + ) + assert col._where_clause(where) == f"WHERE {where.to_sql()}" + + +def test_bbox_covering_tile(covering_collection): + """Tile envelope filter also filters the bbox covering columns.""" + col = covering_collection + tms = morecantile.tms.get("WebMercatorQuad") + tile = tms.tile(13.5, 13.5, 6) + where = col.cql_where(tms=tms, tile=tile) + assert _row_groups_scanned(col, col._where_clause(where)) == "1 / 5" + + content = asyncio.run(col.get_tile(tms=tms, tile=tile)) + assert content diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index f4429525..8b172bef 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -1,6 +1,8 @@ """tipg DuckDB collection models.""" +import math import re +from collections.abc import Iterator from typing import Any import duckdb @@ -19,8 +21,10 @@ mvt_settings = MVTSettings() features_settings = FeaturesSettings() # type: ignore [call-arg] -# Column types (as normalized in the catalog) ST_AsMVT can encode as-is -_MVT_NATIVE_TYPES = frozenset({"real", "float8", "boolean", "text", "json"}) +# Column types (as normalized in the catalog) ST_AsMVT can encode as-is. +# NOTE: `text` is not one of them: unknown DuckDB types (e.g STRUCT, MAP) are +# normalized to `text`, so they must be cast to VARCHAR. +_MVT_NATIVE_TYPES = frozenset({"real", "float8", "boolean", "json"}) # Integer types (as normalized in the catalog) → DuckDB type ST_AsMVT accepts. # The normalized name may come from a type ST_AsMVT rejects (e.g. ``UINTEGER`` @@ -32,6 +36,90 @@ } +# cql2 spatial predicates which imply that the bboxes of the geometries intersect +_BBOX_PREDICATES = frozenset( + { + "s_intersects", + "s_equals", + "s_touches", + "s_within", + "s_overlaps", + "s_crosses", + "s_contains", + } +) + + +def _conjuncts(expr: Any) -> Iterator[Any]: + """Terms of the top-level ``AND`` of a cql2-json expression.""" + if isinstance(expr, dict) and expr.get("op") == "and": + for arg in expr.get("args") or []: + yield from _conjuncts(arg) + else: + yield expr + + +def _coords_bounds(coords: Any) -> list[float] | None: + """[xmin, ymin, xmax, ymax] of (nested) GeoJSON coordinates.""" + if not isinstance(coords, list) or not coords: + return None + + # a position + if all(isinstance(v, (int, float)) for v in coords): + if len(coords) < 2: + return None + x, y = float(coords[0]), float(coords[1]) + return [x, y, x, y] + + bounds = [_coords_bounds(c) for c in coords] + if any(b is None for b in bounds): + return None + + return [ + min(b[0] for b in bounds), # type: ignore [index] + min(b[1] for b in bounds), # type: ignore [index] + max(b[2] for b in bounds), # type: ignore [index] + max(b[3] for b in bounds), # type: ignore [index] + ] + + +def _literal_bounds(literal: Any, srid: int) -> list[float] | None: + """Bounds of a cql2-json geometry literal, in the column CRS (``srid``). + + Literals of columns not in EPSG:4326 are expected to be tagged with their + CRS (``st_setcrs``, see ``DuckCollection._tag_geometry_literal``), other + ones (e.g reprojected with ``st_transform``) are ignored. + """ + if isinstance(literal, dict) and literal.get("op") == "st_setcrs": + literal = (literal.get("args") or [None])[0] + elif srid != 4326: + return None + + if not isinstance(literal, dict): + return None + + if bbox := literal.get("bbox"): + if len(bbox) == 6: + bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] + bounds = [float(v) for v in bbox] if len(bbox) == 4 else None + + elif literal.get("type") == "GeometryCollection": + parts = [_literal_bounds(g, 4326) for g in literal.get("geometries") or []] + if not parts or any(p is None for p in parts): + return None + bounds = _coords_bounds( + [[p[0], p[1]] for p in parts] + [[p[2], p[3]] for p in parts] + ) # type: ignore [index] + + else: + bounds = _coords_bounds(literal.get("coordinates")) + + if bounds is None or not all(math.isfinite(v) for v in bounds): + return None + + return bounds + + def _quote_literal(value: str) -> str: """Quote a SQL string literal.""" return "'" + value.replace("'", "''") + "'" @@ -52,6 +140,10 @@ class DuckCollection(Collection): # Excluded from serialization and repr. connection: duckdb.DuckDBPyConnection = Field(exclude=True, repr=False) + # GeoParquet bbox covering of geometry columns: column name → SQL + # expression of its ``xmin``, ``ymin``, ``xmax`` and ``ymax`` columns. + bbox_coverings: dict[str, dict[str, str]] = {} + model_config = ConfigDict(arbitrary_types_allowed=True) @property @@ -93,14 +185,16 @@ def _select_mvt_property_columns(self, properties: list[str] | None) -> list[str cols.append(f"CAST({q} AS VARCHAR) AS {q}") return cols - def _select_id(self) -> str: + def _select_id(self) -> str | None: """SQL fragment that selects the primary-key column as ``tipg_id``. - Falls back to ``ROW_NUMBER()`` when the collection has no primary key. + Collections without primary key get their ids from the row position + (see ``_run_features_queries``): a ``ROW_NUMBER()`` window would + prevent DuckDB from skipping rows (``OFFSET``) cheaply. """ if self.id_column: return f"{_quote_ident(self.id_column.name)} AS tipg_id" - return "ROW_NUMBER() OVER () AS tipg_id" + return None def _geom_expr( self, @@ -137,7 +231,8 @@ def _select( ) -> str: """Build the SELECT clause for the main features query.""" cols = self._select_property_columns(properties) - cols.append(self._select_id()) + if select_id := self._select_id(): + cols.append(select_id) geom = self._geom_expr(geometry_column, bbox_only, simplify) if geom_as_wkt: @@ -211,44 +306,96 @@ def _from(self) -> str: return f"FROM {self._qualified_name}" def _sort_expressions(self, sortby: str | None) -> list[tuple[str, str]]: - """Parse ``sortby`` into ``(column name, SQL fragment)`` pairs.""" - if sortby: - sorts = [] - for s in sortby.strip().split(","): - parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore - - direction = parts["direction"] - column = parts["column"].strip() - if not self.get_column(column): - raise InvalidPropertyName(f"Property {column} does not exist.") - col_sql = _quote_ident(column) - sorts.append( - (column, f"{col_sql} DESC" if direction == "-" else col_sql) - ) + """Parse ``sortby`` into ``(column name, SQL fragment)`` pairs. + + Without ``sortby``, no sort is applied: sorting means a full scan of + the sort column, and DuckDB preserves the insertion order + (``preserve_insertion_order``), so paging stays stable without it. + """ + sorts: list[tuple[str, str]] = [] + if not sortby: return sorts - if self.id_column is not None: - column = self.id_column.name - else: - column = self.properties[0].name + for s in sortby.strip().split(","): + parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore + + direction = parts["direction"] + column = parts["column"].strip() + if not self.get_column(column): + raise InvalidPropertyName(f"Property {column} does not exist.") + col_sql = _quote_ident(column) + sorts.append((column, f"{col_sql} DESC" if direction == "-" else col_sql)) - return [(column, _quote_ident(column))] + return sorts def _sortby(self, sortby: str | None) -> str: - """Build the ORDER BY clause.""" - return "ORDER BY " + ", ".join(sql for _, sql in self._sort_expressions(sortby)) + """Build the ORDER BY clause (empty without ``sortby``).""" + if sorts := self._sort_expressions(sortby): + return "ORDER BY " + ", ".join(sql for _, sql in sorts) + return "" + + def _bbox_covering_filter(self, where: Expr) -> str | None: + """SQL filter on the GeoParquet bbox covering columns. + + For each spatial predicate (e.g the `bbox` or tile filters) between a + geometry column with a bbox covering and a geometry literal, which is + part of the top-level ``AND`` of ``where``, filter the covering + columns with the bounds of the literal. DuckDB uses the row group + statistics of those columns to skip reading most of the file. + """ + if not self.bbox_coverings: + return None + + filters = [] + for expr in _conjuncts(where.to_json()): + if not isinstance(expr, dict) or expr.get("op") not in _BBOX_PREDICATES: + continue + + args = expr.get("args") or [] + if len(args) != 2: + continue - @staticmethod - def _where_clause(where: Expr | None) -> str: + for prop, literal in (args, args[::-1]): + if not isinstance(prop, dict) or "property" not in prop: + continue + + name = prop["property"] + covering = self.bbox_coverings.get(name) + column = self.get_column(name) + if not covering or not column: + continue + + if bounds := _literal_bounds(literal, column.srid or 4326): + xmin, ymin, xmax, ymax = bounds + filters.append( + f"{covering['xmin']} <= {xmax!r} AND {covering['xmax']} >= {xmin!r} " + f"AND {covering['ymin']} <= {ymax!r} AND {covering['ymax']} >= {ymin!r}" + ) + break + + return " AND ".join(filters) or None + + def _where_clause(self, where: Expr | None) -> str: """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" if where is None: return "WHERE TRUE" - return f"WHERE {where.to_sql()}" - def _row_to_feature(self, row: dict[str, Any], geom_as_wkt: bool) -> Feature: - """Convert a result row to a Feature.""" + sql = where.to_sql() + if covering := self._bbox_covering_filter(where): + sql = f"{covering} AND ({sql})" + + return f"WHERE {sql}" + + def _row_to_feature( + self, row: dict[str, Any], geom_as_wkt: bool, position: int + ) -> Feature: + """Convert a result row to a Feature. + + ``position`` (1-based row position in the matched features) is used as + id for collections without primary key. + """ g = row.pop("tipg_geom") - id = row.pop("tipg_id") + id = row.pop("tipg_id", position) if g is not None and not geom_as_wkt: g = orjson.loads(g) @@ -261,25 +408,37 @@ def _row_to_feature(self, row: dict[str, Any], geom_as_wkt: bool) -> Feature: return Feature(type="Feature", geometry=g, id=id, properties=row) def _run_features_queries( - self, count_sql: str, sql: str, geom_as_wkt: bool - ) -> tuple[int, list[Feature]]: - """Run the COUNT and Features queries (blocking).""" + self, + count_sql: str | None, + sql: str, + geom_as_wkt: bool, + offset: int, + limit: int, + ) -> tuple[int | None, list[Feature], bool]: + """Run the COUNT (if any) and Features queries (blocking). + + ``sql`` must select up to ``limit + 1`` rows: the extra row tells if + there are more features. Returns ``(matched, features, has_more)``. + """ cursor = self.connection.cursor() try: - logger.debug(count_sql) - matched = cursor.execute(count_sql).fetchone()[0] # type: ignore [index] + matched = None + if count_sql: + logger.debug(count_sql) + matched = cursor.execute(count_sql).fetchone()[0] # type: ignore [index] logger.debug(sql) cursor.execute(sql) names = [d[0] for d in cursor.description] # type: ignore [union-attr] + rows = cursor.fetchall() features = [ - self._row_to_feature(dict(zip(names, r)), geom_as_wkt) - for r in cursor.fetchall() + self._row_to_feature(dict(zip(names, r)), geom_as_wkt, offset + i) + for i, r in enumerate(rows[:limit], start=1) ] finally: cursor.close() - return matched, features + return matched, features, len(rows) > limit def _run_tile_query(self, sql: str, layer: str) -> bytes: """Run the MVT query (blocking).""" @@ -334,7 +493,13 @@ async def features( ) where = self._where_clause(where_filter) - count_sql = " ".join(["SELECT COUNT(*)", self._from(), where]) + # Without filter, DuckDB answers COUNT(*) from the table/file metadata. + # With a filter it is a full scan (while the features query can stop + # after `limit` rows), so `numberMatched` is not returned. + count_sql = None + if where_filter is None: + count_sql = " ".join(["SELECT COUNT(*)", self._from()]) + sql = " ".join( [ self._select( @@ -347,20 +512,20 @@ async def features( self._from(), where, self._sortby(sortby), - f"LIMIT {int(limit)}", + f"LIMIT {int(limit) + 1}", f"OFFSET {int(offset)}", ] ) - matched, features = await run_in_threadpool( - self._run_features_queries, count_sql, sql, geom_as_wkt + matched, features, has_more = await run_in_threadpool( + self._run_features_queries, count_sql, sql, geom_as_wkt, offset, limit ) returned = len(features) return ItemList( items=features, matched=matched, - next=offset + returned if matched - returned > offset else None, + next=offset + returned if has_more else None, prev=max(offset - limit, 0) if offset else None, ) @@ -407,7 +572,7 @@ async def get_tile( ) sorts = self._sort_expressions(sortby) - order_by = "ORDER BY " + ", ".join(sql for _, sql in sorts) + order_by = self._sortby(sortby) inner_sql = " ".join( [ @@ -514,49 +679,97 @@ def _geometry_srid(data_type: str) -> int: _PARQUET_VIEW = re.compile( - r"^CREATE VIEW .+? AS SELECT \* FROM read_parquet\('(?P(?:[^']|'')*)'\);?$", + r"^CREATE VIEW .+? AS SELECT \* FROM read_parquet\('(?P(?:[^']|'')*)'" + r"(?P, \(file_row_number = true\))?\);?$", re.DOTALL | re.IGNORECASE, ) +# Column added by `read_parquet(..., file_row_number = true)` +FILE_ROW_NUMBER = "file_row_number" -def _parquet_view_source(view_sql: str | None) -> str | None: - """Path (or glob) of a view defined as ``SELECT * FROM read_parquet('')``.""" + +def _parquet_view_source(view_sql: str | None) -> tuple[str, bool] | None: + """Path (or glob) of a view defined as ``SELECT * FROM read_parquet('')`` + and whether the ``file_row_number`` option is set (as done for file sources, + see ``tipg.duck.database``). + """ if view_sql and (m := _PARQUET_VIEW.match(view_sql.strip())): - return m.group("path").replace("''", "'") + return m.group("path").replace("''", "'"), bool(m.group("row_number")) return None +_COVERING_KEYS = ("xmin", "ymin", "xmax", "ymax") + + +def _parse_covering(meta: dict[str, Any]) -> dict[str, str] | None: + """SQL expressions of a GeoParquet bbox covering (e.g ``"bbox"."xmin"``).""" + covering = (meta.get("covering") or {}).get("bbox") or {} + exprs = {} + for key in _COVERING_KEYS: + path = covering.get(key) + if not path or not all(isinstance(p, str) for p in path): + return None + exprs[key] = ".".join(_quote_ident(p) for p in path) + + return exprs + + def _parse_geo_metadata(geo: str) -> dict[str, dict[str, Any]]: - """Bbox and geometry types per column, from a GeoParquet ``geo`` metadata value.""" + """Bbox, geometry types and bbox covering per column, from a GeoParquet + ``geo`` metadata value (each one is ``None`` when unknown). + """ columns: dict[str, dict[str, Any]] = {} for name, meta in orjson.loads(geo).get("columns", {}).items(): bbox = meta.get("bbox") if not bbox or len(bbox) not in (4, 6): - continue - + bbox = None # 3D bbox: [xmin, ymin, zmin, xmax, ymax, zmax] - if len(bbox) == 6: + elif len(bbox) == 6: bbox = [bbox[0], bbox[1], bbox[3], bbox[4]] types = meta.get("geometry_types") columns[name] = { - "bbox": [float(v) for v in bbox], - "geometry_types": set(types) if types else None, + "bbox": [float(v) for v in bbox] if bbox else None, + "geometry_types": set(types) if types and bbox else None, + "covering": _parse_covering(meta), } return columns +def _merge_geo_metadata(col: dict[str, Any], other: dict[str, Any]) -> None: + """Merge the GeoParquet metadata of a column from another file (in place).""" + if col["bbox"] is None or other["bbox"] is None: + col["bbox"] = None + col["geometry_types"] = None + else: + col["bbox"] = [ + min(col["bbox"][0], other["bbox"][0]), + min(col["bbox"][1], other["bbox"][1]), + max(col["bbox"][2], other["bbox"][2]), + max(col["bbox"][3], other["bbox"][3]), + ] + if col["geometry_types"] is None or other["geometry_types"] is None: + col["geometry_types"] = None + else: + col["geometry_types"] |= other["geometry_types"] + + if col["covering"] != other["covering"]: + col["covering"] = None + + def _geoparquet_metadata( cursor: duckdb.DuckDBPyConnection, path: str, ) -> dict[str, dict[str, Any]]: - """Bbox and geometry types of geometry columns, from GeoParquet metadata. + """Bbox, geometry types and bbox covering of geometry columns, from + GeoParquet metadata. - Only the file footers are read. For multiple files (glob), bboxes and - geometry types are merged, and a column is only returned when every file - has a bbox for it. Geometry types are ``None`` when unknown. + Only the file footers are read. For multiple files (glob), the metadata + is merged: a column is only returned when every file describes it, its + bbox (and geometry types) is ``None`` when one file has no bbox and its + covering is ``None`` when files have different coverings. """ try: nfiles = cursor.execute("SELECT count(*) FROM glob($1)", [path]).fetchone()[0] # type: ignore [index] @@ -586,19 +799,8 @@ def _geoparquet_metadata( for name in list(merged): if (other := file_columns.get(name)) is None: del merged[name] - continue - - col = merged[name] - col["bbox"] = [ - min(col["bbox"][0], other["bbox"][0]), - min(col["bbox"][1], other["bbox"][1]), - max(col["bbox"][2], other["bbox"][2]), - max(col["bbox"][3], other["bbox"][3]), - ] - if col["geometry_types"] is None or other["geometry_types"] is None: - col["geometry_types"] = None else: - col["geometry_types"] |= other["geometry_types"] + _merge_geo_metadata(merged[name], other) return merged or {} @@ -640,7 +842,7 @@ def _geometry_metadata( for c in columns: if c["type"] == "geometry": c["geometry_type"] = "Geometry" - if meta := geo_metadata.get(c["name"]): + if (meta := geo_metadata.get(c["name"])) and meta["bbox"]: _apply_geo_metadata(cursor, c, meta) @@ -726,9 +928,11 @@ def _collection_index( # noqa: C901 ): continue + parquet_source = _parquet_view_source(view_sql) + geo_metadata = {} - if settings.spatial_extent and (path := _parquet_view_source(view_sql)): - geo_metadata = _geoparquet_metadata(cursor, path) + if settings.spatial_extent and parquet_source: + geo_metadata = _geoparquet_metadata(cursor, parquet_source[0]) _geometry_metadata(cursor, columns, geo_metadata) @@ -754,6 +958,13 @@ def _collection_index( # noqa: C901 id_column = column break + # Parquet file source: row position in the file + if id_column is None and parquet_source and parquet_source[1]: + for column in cols: + if column.name == FILE_ROW_NUMBER: + id_column = column + break + datetime_column = None geometry_column = None for column in cols: @@ -777,6 +988,11 @@ def _collection_index( # noqa: C901 id_column=id_column, datetime_column=datetime_column, geometry_column=geometry_column, + bbox_coverings={ + name: meta["covering"] + for name, meta in geo_metadata.items() + if meta["covering"] + }, connection=connection, ) diff --git a/tipg/duck/database.py b/tipg/duck/database.py index 89f6f15b..da8d64a2 100644 --- a/tipg/duck/database.py +++ b/tipg/duck/database.py @@ -16,10 +16,25 @@ def _quote_literal(value: str) -> str: return "'" + value.replace("'", "''") + "'" -def _source_query(path: str) -> str: - """SQL `SELECT` for a file source.""" - if path.lower().endswith((".parquet", ".parq", ".geoparquet")): - return f"SELECT * FROM read_parquet({_quote_literal(path)})" +def _is_parquet(path: str) -> bool: + """Is the source a parquet file (or glob).""" + return path.lower().endswith((".parquet", ".parq", ".geoparquet")) + + +def _is_glob(path: str) -> bool: + """Is the source a glob pattern (multiple files).""" + return any(c in path for c in ("*", "?", "[")) + + +def _source_query(path: str, file_row_number: bool = False) -> str: + """SQL `SELECT` for a file source. + + With `file_row_number`, parquet sources get a `file_row_number` column + (row position in the file), used as stable feature id. + """ + if _is_parquet(path): + options = ", file_row_number = true" if file_row_number else "" + return f"SELECT * FROM read_parquet({_quote_literal(path)}{options})" return f"SELECT * FROM ST_Read({_quote_literal(path)})" @@ -51,6 +66,10 @@ def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: conn.execute("SET GLOBAL geometry_always_xy = true") conn.execute("SET GLOBAL TimeZone = 'UTC'") + # Cache parquet files metadata (footers) in memory, so each query doesn't + # fetch it again (expensive for remote files with many row groups). + conn.execute("SET GLOBAL parquet_metadata_cache = true") + # Attach the main database file if specified in settings. if settings.database: # use the file name (without extension) as alias, also for URLs @@ -63,9 +82,20 @@ def create_connection(settings: DuckDBSettings) -> duckdb.DuckDBPyConnection: # Attach and register file sources as views in the in-memory database. for name, path in settings.sources.items(): logger.debug(f"Registering DuckDB source {name}: {path}") - conn.execute( - f"CREATE OR REPLACE VIEW memory.main.{_quote_ident(name)} AS {_source_query(path)}" - ) + view = f"CREATE OR REPLACE VIEW memory.main.{_quote_ident(name)} AS" + + # Single parquet file: add the row position in the file as a stable + # feature id (looking up a row position only reads one row group). + # Not for globs, where row positions are not unique. + if _is_parquet(path) and not _is_glob(path): + try: + conn.execute(f"{view} {_source_query(path, file_row_number=True)}") + continue + except duckdb.BinderException as e: + # e.g the file already has a `file_row_number` column + logger.debug(f"Could not add `file_row_number` to {name}: {e}") + + conn.execute(f"{view} {_source_query(path)}") return conn diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py index 5732b8fb..ad97465d 100644 --- a/tipg/duck/settings.py +++ b/tipg/duck/settings.py @@ -43,7 +43,9 @@ class DuckDBCatalogSettings(BaseSettings): for sources matching many (remote) files. When disabled, or when the metadata is missing, collections have no spatial extent (world bbox) and a generic `Geometry` type. Tables and other views - never have a spatial extent. + never have a spatial extent. The GeoParquet bbox `covering` + columns (if any) are also read from this metadata and used to + speed up spatial filters (`bbox`, tiles, cql2 spatial predicates). """ From 4bcbaa3c5aaab3b7bbef3e5fe4d2617a5c828968 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 12 Nov 2026 15:04:04 +0100 Subject: [PATCH 18/22] perf: improve perf and support String CRS --- docs/src/advanced/duckdb.md | 3 +- examples/duckdb/build_source_coop_db.py | 201 ++++++++++++++++++ examples/{ => duckdb}/main.py | 0 examples/duckdb/source_coop.duckdb | Bin 0 -> 274432 bytes examples/duckdb/source_coop_geoparquet.md | 123 +++++++++++ tests/duck/test_geoparquet.py | 191 +++++++++++++++++ tipg/collections.py | 43 +++- tipg/duck/collections.py | 241 +++++++++++++++++++--- tipg/duck/settings.py | 3 + tipg/pg/collections.py | 6 +- 10 files changed, 765 insertions(+), 46 deletions(-) create mode 100644 examples/duckdb/build_source_coop_db.py rename examples/{ => duckdb}/main.py (100%) create mode 100644 examples/duckdb/source_coop.duckdb create mode 100644 examples/duckdb/source_coop_geoparquet.md diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index afad90fc..65146693 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -26,6 +26,7 @@ python -m pip install "tipg[duckdb]" | `exclude_tables` | `TIPG_DUCKDB_EXCLUDE_TABLES` | | Tables (`schema.table`) to exclude | | `only_spatial_tables` | `TIPG_DUCKDB_ONLY_SPATIAL_TABLES` | `True` | Only expose tables/views with a geometry column | | `spatial_extent` | `TIPG_DUCKDB_SPATIAL_EXTENT` | `True` | Read the spatial extent and geometry type of parquet sources from their GeoParquet metadata (file footers only). For a glob, every matched file's footer is read on each catalog refresh: disable it for sources matching many remote files | +| `metadata_threads` | `TIPG_DUCKDB_METADATA_THREADS` | `16` | Number of parquet sources whose metadata is read concurrently when building the catalog | Table level configuration (`TIPG_TABLE_CONFIG__...`, fallback key names, column sorting) is shared with the PostgreSQL backend (see [Configuration](../user_guide/configuration.md)). @@ -156,7 +157,7 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) ### Differences with the PostgreSQL backend -- **CRS**: the SRID of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. +- **CRS**: the CRS of a geometry column is read from its type (e.g `GEOMETRY('EPSG:26915')`). Columns without CRS (plain `GEOMETRY`) or in `OGC:CRS84` are assumed to be in `EPSG:4326`. CRS without EPSG code (e.g PROJJSON from GeoParquet metadata, or `IGNF:LAMB93`) are supported: the CRS definition is used as is for reprojection (`Column.srid` is then a string), and the collection `crs` is the OGC URI of the CRS authority code (e.g `http://www.opengis.net/def/crs/IGNF/0/LAMB93`), or `None` when the CRS has no authority code. PROJJSON definitions of EPSG CRS are matched to their EPSG code. - **Extents and geometry type**: tables and views are never scanned to build the catalog. The spatial extent and geometry type of parquet `sources` are read from their [GeoParquet](https://geoparquet.org) metadata (`bbox` and `geometry_types`, in the file footers). Other collections have no spatial extent and a generic `Geometry` type, and no collection has a temporal extent. - **Items order**: without `sortby`, features are returned in their insertion order (no default sort on the primary key). - **Feature ids**: single parquet file `sources` are read with `read_parquet(..., file_row_number = true)`, and the added `file_row_number` column (row position in the file, starting at 0) is used as id when the collection has no other id column (`TIPG_TABLE_CONFIG__..._pk` or fallback key names). Looking up a feature by id (`/items/{itemId}`) then only reads one row group. Other collections without primary key use the feature position in the response as `id`, and don't support `/items/{itemId}`. diff --git a/examples/duckdb/build_source_coop_db.py b/examples/duckdb/build_source_coop_db.py new file mode 100644 index 00000000..a6484cd2 --- /dev/null +++ b/examples/duckdb/build_source_coop_db.py @@ -0,0 +1,201 @@ +"""Create a DuckDB database with one view per Source Cooperative GeoParquet file. + +The sources are listed in `source_coop_geoparquet.md` (markdown table with the +product name, title and an example file of each Source Cooperative product). + +Views read the remote files directly (no data is copied), using the same form +as tipg file `sources`: + + CREATE VIEW main. AS + SELECT * FROM read_parquet('https://data.source.coop//', file_row_number = true) + +So the tipg DuckDB catalog reads their GeoParquet metadata (spatial extent, +geometry type, bbox covering) and uses `file_row_number` as feature id. + +The remote file footers are read in parallel (`--threads`), then the views are +created sequentially from DuckDB's parquet metadata cache. + +Usage: + + python build_source_coop_db.py [source_coop_geoparquet.md] [source_coop.duckdb] [--threads 16] + + TIPG_DUCKDB_DATABASE=source_coop.duckdb uvicorn main:app + +""" + +import argparse +import pathlib +import re +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import duckdb + +BASE_URL = "https://data.source.coop" +HERE = pathlib.Path(__file__).parent + + +def quote_literal(value: str) -> str: + """Quote a SQL string literal.""" + return "'" + value.replace("'", "''") + "'" + + +def quote_ident(value: str) -> str: + """Quote a SQL identifier.""" + return '"' + value.replace('"', '""') + '"' + + +def parse_table(path: pathlib.Path) -> list[dict[str, str]]: + """Parse the rows of the markdown table. + + Columns: # | Product | Title | Source Cooperative | Example file | ... + """ + rows = [] + for line in path.read_text().splitlines(): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if len(cells) < 5 or not cells[0].isdigit(): + continue + + rows.append( + { + "product": cells[1].strip("`"), + "title": cells[2], + "file": cells[4].strip("`"), + } + ) + + return rows + + +def view_name(product: str) -> str: + """View name from the product name (e.g `fiboa/es-an` -> `fiboa_es_an`).""" + return re.sub(r"[^0-9a-zA-Z]+", "_", product).strip("_").lower() + + +def create_view( + conn: duckdb.DuckDBPyConnection, name: str, url: str, title: str +) -> list[str]: + """Create a view reading a remote parquet file, return its column types. + + Binding the view reads the remote file (HTTP requests), which is the + slow part. Thread safe: uses its own cursor (connection). + """ + cursor = conn.cursor() + try: + view = f"CREATE OR REPLACE VIEW main.{quote_ident(name)} AS" + try: + cursor.execute( + f"{view} SELECT * FROM read_parquet({quote_literal(url)}, file_row_number = true)" + ) + except duckdb.BinderException: + # the file already has a `file_row_number` column + cursor.execute(f"{view} SELECT * FROM read_parquet({quote_literal(url)})") + + cursor.execute( + f"COMMENT ON VIEW main.{quote_ident(name)} IS {quote_literal(title)}" + ) + return [ + r[1] + for r in cursor.execute(f"DESCRIBE main.{quote_ident(name)}").fetchall() + ] + finally: + cursor.close() + + +def main(md_path: pathlib.Path, db_path: pathlib.Path, threads: int) -> None: + """Create the database.""" + rows = parse_table(md_path) + print(f"{len(rows)} sources") + + conn = duckdb.connect(str(db_path)) + conn.execute("INSTALL spatial") + conn.execute("LOAD spatial") + conn.execute("SET parquet_metadata_cache = true") + # cache HTTP HEAD responses (file size, last modified) too + conn.execute("SET enable_http_metadata_cache = true") + + sources = { + view_name(row["product"]): (row, f"{BASE_URL}/{row['product']}/{row['file']}") + for row in rows + } + + created, failed, nongeo, conflicts = [], [], [], [] + + def _done(name: str, types: list[str]) -> None: + if not any(t.upper().startswith("GEOMETRY") for t in types): + nongeo.append(name) + created.append(name) + print(f"ok {name}", flush=True) + + def _failed(name: str, url: str, e: Exception) -> None: + error = str(e).splitlines()[0] + failed.append((name, url, error)) + print(f"FAILED {name}: {error}", flush=True) + + # Create the views in parallel (each one in its own transaction) + start = time.time() + with ThreadPoolExecutor(max_workers=threads) as pool: + futures = { + pool.submit(create_view, conn, name, url, row["title"]): name + for name, (row, url) in sources.items() + } + for future in as_completed(futures): + name = futures[future] + try: + _done(name, future.result()) + except duckdb.TransactionException: + # concurrent catalog writes conflicted: retry sequentially + conflicts.append(name) + except duckdb.Error as e: + _failed(name, sources[name][1], e) + + for name in conflicts: + row, url = sources[name] + try: + _done(name, create_view(conn, name, url, row["title"])) + except duckdb.Error as e: + _failed(name, url, e) + + conn.close() + + print(f"\ndone in {time.time() - start:.0f}s ({len(conflicts)} retried)") + print_report(created, failed, nongeo) + + +def print_report( + created: list[str], failed: list[tuple[str, str, str]], nongeo: list[str] +) -> None: + """Print the summary.""" + print(f"\n{len(created)} views created, {len(failed)} failed") + if nongeo: + print( + f"no GEOMETRY column (hidden by `only_spatial_tables`): {', '.join(nongeo)}" + ) + for name, url, error in failed: + print(f" {name}: {url}\n {error}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "sources", + nargs="?", + type=pathlib.Path, + default=HERE / "source_coop_geoparquet.md", + help="markdown table of the sources", + ) + parser.add_argument( + "database", + nargs="?", + type=pathlib.Path, + default=HERE / "source_coop.duckdb", + help="output DuckDB database", + ) + parser.add_argument( + "--threads", + type=int, + default=16, + help="number of files read concurrently (default: 16)", + ) + args = parser.parse_args() + main(args.sources, args.database, args.threads) diff --git a/examples/main.py b/examples/duckdb/main.py similarity index 100% rename from examples/main.py rename to examples/duckdb/main.py diff --git a/examples/duckdb/source_coop.duckdb b/examples/duckdb/source_coop.duckdb new file mode 100644 index 0000000000000000000000000000000000000000..1e465511f1e6c053554a3301856f7d484f860fc7 GIT binary patch literal 274432 zcmeIbdvGL4ejZl4gZE&b`@Y?~ZH|t-U3z;O-FVCZdrPthXkf657tz2jcaO(np}Wvs zgMM9AH!xVn(GN*hkP<~YEQ+S6kVu)NLZ(QQqC{DyEHUCp$KKpgGAT<`NR-2tZ8Bs> z$ba|``ya>n`(^R`Mp2B zr1bT7oVz`9dnWqQ^88%s;6R@(A4Xq8Xa29BegB6ue`e{=&)0rn03Jr4ZY`HIrN z{V&J=UDRYsA|Mfv2uK7Z0ulj7Iye&?C z;jPr6QPWkkd7?Ht^#k2Zy_0I29sSKCJXu+J#1F>Z8R!Y>W!6VUANjs>CIF+RcKUdTBDqz{~u^I-Fp0(@Va!F*IcEV zGwiN=t4}~Ye*B3N%6z1ZvkGoe4W_X!tl$k!x`HzxPX=CmFZn3zR z+R%^nT52w}xxbrQt2Ga_T57e~X@J^Bvym$5TB_JF4>b^NPp`LhQ)_ojJ(X{0wNuLg z(~jqEXWxus)MqqL?fr;Q>+f;;k@BjSU^;7gBSb6o1@z}Vi>$KZxune7=_lk29jTi7 z;WPH{6y=hs*F1+K?bDX-IUa2r?V6jx??uHsWsWM!w8kD^FROUN^ke7bc=(2a5^+KB zmSz*35*6cl1o4!QALDMOX0>&5=>Q#A@G|GdsMT1bNgXm3gpP)OIZ4NA@OBpZVfKVC z@vFv~hliHVo*(n~>DfM86@S&dIStZ8u3NgM8YNvlG&OXd&BoL5_)jsZE|YB0 z8qGTTVF^}GMp3gA(jrAE_XFLOIN|AeSszFzJb9u&^W{0$G3gWdaQbDEKo9E%YtVM@ zdKx%>Sl1rXJ9N)`j(P2!^2Si!+Dd+du4lb7`edOMl$w=_&bn7|VuA|}qv^SLyjd=# zq40RJ#vW>>sj*gM|FAmW(A%Z5D>r#97;-%|+|LkjKr33!UG}ND?^czsBvf-hs4DLX zHFrH)&GqOVem%*5WNS_Id}L#g^3BxSpAV|Hlo6}m{>H%S?W;^Hn6}zlNj=t03w_`< zcA!6O>kT@zo;AwXTV}HaQw>JWQjHY1o8Angp=%X=v}QDptTD}kOt+>=&00-oUFrms z0ljV1^{Xh?Y^(O+wB9H?2a_o8!IIvxD&CSsh4{&XgXY62c9~VxTKZL%Y@1rcB9lxs z#wFaN-LanYR1Z3&QG?;<&9m$95^usw!Xj&R4r+!~)ypGvGw%n zboVQ)8vSeXzu#MvFC8@XcH5}HIBZzDdCat7wbiT{rPIN6dArn%x70K*HM8D0tnu6I z>g5i5Woo|ER(J45VEnSbXG?9%ZnFK_%(Kg$M@a3kM@SW0cnqQ$uUyWr6}FZ?RJZb*h0FK$;kw#q>FJ%_?X}(f zrgK2QUP-(V=1LXVCjlF&P+v^qtPl5sc&w{4Z~Lfwl$ zhxI2+hntS?R9vl(T^#rYI|Q@6UIn*j>Dj6D)OHqpoxnR)#Yor(KtWm-eLz0G4_NY%7~@;&kaFywbZEKb(TjzEaZ39QO4CV&iC>Ct@S;1ePwcWeOK^zeQQ|- zxyILbKUm*dUzy5pJ&f7!}U7Kbe(u6|5l$& zu4)GcG$e>o(=^IW(>eoMi!#$rnVC+b(t$wN)w)rE;|@w>QjULyI9@Xj_0)Y{kjFBaDw7LC39lIwpA|A#VPyAU7!^ zmYXD}2XnAbURqsW887Y>me&i#(cS#kT4CgVVRuEafG8o7Di~oWk?n}RczXQ(KI!qG zLxVT^+{_u!!YM0LHY}}tOdk4*UItC-b@Fo>dSaFO-g3>TgI7$r6`;&F>qaT{F0XW| zSTc3JLCW=Bvsp{M!9-r#FY+Qi<~JRBIQ1shbjfCHVbU+4IWrC0rHqFyKYsIvZhhW@ z_qMLxTDoPoU$UHTH_b|hpRqE>Gq>+#DEEPd9)suJru411Zjmz?zhy>A2?iL(w~*V= z>Xf@*%&V|AU^jJ|9ZSvEFpy`dg|Bq98nqdXPGV$)^a9(+#g7j(OIJ(i0+)WsE@&&i za(5P|VB>Y_lrL_(rZ>y5VN1LEp>FCJ{aZEZ&Q1lknr6G2%GY%hj1y7r)Hs@g*Xm06g*b2mGC=dD{tgX~}UCef9`7RQUd zwUk@DdpkFOCzs1*XR=@Y>RY!kh-ku~qR!Sjg}a{B+UgvLiaR&gYPc7_!IzxJwI3@* z)>pezrYF)44ov;nV7)e=-O$>|EJMCC%MdvYdRF978^&Q-ub|o8`s!OMd5sU_-dnFXIt8a?SuQ?pfF=uHo7eUdpYdx!l4cM}px6VdDXMu$F;QxoXqKUUE~2PLnhTcOdV=9OHW~Nx?)gEKh|m;JXUTt z2yyH!@7~Q(n{ZylhK+ypUDAsSU2h}o{!5@hB=EQ$J)wI$&ogDMb_%W-4Ck5U6hQ!Q z5Vb)2;TCVtcfD`S{br-?m%|u)kNeHd4|cyf@BIpBq13x-?TA-4<2(`#>k^q_+6WL_ z@ZN!SBWHe#J74|km*abiW%%ZF%h0F^2Kx!|Zs8?K^zIHMU;dbK$oPKQ$=XFI);creu~3@#=c)n!8$|$h>qiY zO*_!)+S?$q@o}@k5=U?&YS!B+Sd0d&!-j#}?#C!VT>M zE_%n59zJ!x3%iqsrPlWj>kg60NSkB4>UXlOeVVF2ME0Jo-)u< z;J~Icr_hrc6>2Et7a{Qghi|*8I%01rH+Oq3`_`=!l%dw_Zu8bsI*Su`79r;e1JCty zi??U*zIBU)mhbw7h1-kS?lP;myN(KQnU$T%&CEh-apgP4q4&1WmcLh3Ith)3?W(dZ zjE7BI`GSVy523R4zl@{*#p|sh+u>(}wgY9v+75qbNZSFPvh|hh%p~Le7NqUiG8{wg z(}@+KcuyDa=XVN@b=pXP(rEE!Nak2^Jli@|QCVTUuYBzUoDU$L3o z%*;%t`2IS5L$ifNPmi3~&MvYlr{4EgnR+e#VU(TyS-YjE-5!^WhTc~^Qg&H`_qx0S zmIOO=Es1Z5l7%Ohf#{v~<9nw8xDe{4 zem&SrQATVp^*aOiQo9qj_fh!ySJ!unlgk)0P$@N0+*{tJgl7c7Gaw*_7e?2z?`L=E zGsiw>+2nevcjER^ppqhTDl_ep!xRhC8!t+?)M+mS>|9EF0Aj-~898ia4HUm(T6(jO-Wr zZ&0ySQ8Dp7E57GedBxZvDki>j;+tOvB-mxSJ(i7%iSHcCp5@u77|TY*#5eynDkhHS z+2wP4t5mVosl8^q)ofAMJC3AQt9IDy*}^moeZJvP&wJ7O;gxZ+3{|5%QbNNUg@>T6U#6sLbiL*NtC&k3 zDK~|}=p_p|0+;4ya|^a1)lHF1p4T;A1Im&wUc)}N<2CGKJ6?kw({EzTyEPw$x2j4b zp#t@#s&Zc_P^-}jRFd{Uwf2Q&{6bJ;qKsIL>F*D$G2KMWnKQgIO+x~3zo_KxzXJDy z0T!OJqswp%!LtBIP36?Nj9lO2%p5XLZ`i*rQ`8zfM<&IWvC_~Ph-fs5Zev+?MA{U{ zk2_GLmnR5Jg&;GVaL8P4A#hDE!5gBNox4&;b3-e!`$F8QRL^GPIwk?|5d)FIBFo}? zjbD%HHLA@TCK{<)8IFy!=`VPRm`Q~ay*&j7a)80A&ut^tE0w46=!HQVBl5zP-cen} z%cY2CqfR37I+Y__S%kEmoMRe{s z51zHg#i-+GsRqCGD0SA>81DrymyKh1xuM!10H=*%V4X!B$9IE&ufJ|k?;t!GL>tC< z5wFB4D&wTmRgE0aWHzLmjap*+WwLTkyZ`uLW0{%?jorLsmeXcM$so72*t9GcPtx#ipHNy_2I4P7#j+$FX^e zI9~Rx8dme2{Awn@n$$XYVU0aRipl+Z_CrLq`On&IzU!k?`|o`Pn+^>@AcC-&$n$cc zhiTB06(N7-UrZW4{ZUn^3R)b-(&9|sP>3+U9V858^dgL~`$yS??td{bamKH5d%K}R zF~^{yM_0Le*s0ajRt^3X8Uu4EG!Cy&8JfmrPy36f)vxwPtHxoLsj#Vb_M(5XyATM7M=I zb_(_~nM6mVI!1`c;&3kNWfwUYp`;N^BxB}Iw?c-#6Kdc8pOBCNS^1Py*0Y8IO^(7` zC*PD*95Wk~S$uTfX!y??4fi}jQdZ7F2?(u;k~Cf$TdwYW`aTZ+t#bb7SG6o5aB z>E&wLsR~0BY|g6l@D&;@b%@1KsH6;iq>r=ddBjv3E%a(y4N(}`Y$iK1Q>tgo%7I}* zx2a@iXJ+osXX+~Zp2?~WUCl12S(PrHxtpC&+84gDm0#aUk=H1tV~wJ6Sudwf42qya zjZ(+i+w-Y65VM<$9L;)xA;CU8a8M7~{mdf;t2lj;$mHq0gxf1d;D49TsKi{vpGvAe z?_v;3Nc?8B#6Oo865kkIz)+L7(Q~XVn85l3}|Gz|JuxMzj zq1Voi4^D|0JC(L_=~_;Maapl4+01@sxd^Uc#$OgeOS!z7-Cr((54kyTXI`CAH*)!? z{G@F8LQyDD_7{N>N5ErT*WGrqsXcO(`8?N7ags2{^jS(ij}pAG(|!T&wn!oat6Fr_oGy zZDo@YC_R(KKT6K?WHP>R!+#8G+`U+!XX3GcsRENT)cZuyB;~1SR|tqMu@^m3M#{?T zQIe*oD%aDCv+m>6PF4BQq@4P4RVfQf9Yiw9gTZhwD@2p81!+PVu{8PT1Jh&#THU16 zejPCsFMEEnH&G6agQlix#+-FHI44*k8CD4YOxDH^mBm#po#c= zHl6N$LvLfuBK1aoeXfYrjcTj#`bP%{UG_W}wRG<>^R_Sp^EGPOEX6)N?Rkp%ClR?Y zcbC6=u|5#Hu4mJp^EGV*hkwKrl>XOigoK$BEjt(4vLhi6wbUODs7N@DZKppma66qq zJ4LWgv(uVVTc%!S>u=gaQckmd5wrC$TY%=p(Wny*U@iQ@=iww6(uPgH0*c^1C_UAlOE=RQD zU%YX?6FWE8)>9d~`l$`@6;3Of#u1*niA!+;Z#vP$%}t8Y*UU6#@E`)LRmG%{D(0y) zFw?8up*uMfC_R%3@6qg(S}J>YE{iL=P2jvUyLjhzb{=!wx=-R<#L3*INsUqI# zjSI6`Hji+qiJs0(K-4!dg$uJ?2sh6JPv1;TX9{ed&FMF3iWRP>(OpcqT0vA1`bx}+ zVv|5=dW$^?Q^mSV?WTEaY3BA!_Rie=Y;G|(pIuy>o9#M@uS>;ZW}co;&AyEJc%G;5 zy!B3l9=eQ*@_xf}N;FKGCx-dh_yp+)mN#<6oj37=m}!8R2t3qO(YZVB?8b9JCH4sh zn0cC=y*=ywv*%7eu?HA*F6$hohw-`4Pps-)8AX$R zcIH{SsyMuvUX7cbaE3&@%e2`e4|X)5eQZYy+Q)WuAUh@{ImS^MTA`_wP>p}Ls=OoA z__yQK_>fX5#{0)z@U-}g3+*AP(_wJ;lGol2QN{~08fz= z9-%OU2qpg}3t`0%csL9F@TyaUO2cw}#^C-^Uur*Bt%p|oc`+IB{0J6iVZ$FI90(Rb zHmL>hT336wPcy(a7F;uc=R8uTe4}f^goPG+hc_Otk-+%n((QHP!{;2%^)zrp%t#s# z4`##%#0vwrKios)_)9@@P(~~{esN%OOgW}4STc^*p~6|mSm2?|b0S|o&_C9(SU>~o zaM=OqAFD^0R*#JFLkfF0$`R~|w}$_VerR!^tH-s_OKOea1PdZJ1j7J(zV1fsm@xj_Z_`u`> ziz?$Ok`Mzo@4oz*Xb&ga8MN*bjT-k!;)z(_-nDMpN3+Yra zNBe94C4w!n6f7nnq-dcRGLY|SRrvqeyim}_)IK(9QLKSvBroZK{p`e?VBRC;x=%&o z>^b`hiakZfU7yH61m;0Pk%2X%zbHh;Qk=+$et{vL{Af^cP)4lap!qF>Ei8H6VG8L? z2)sC!YyR{{6IqT z!uYu>$h#0Gudw?k#J+jF7Zz_N*ib?Q{xd-WQ${R-|MI{Demz3Bn?{9OZ2qsRH<%LW zFntn>km>ByB&JW9-9B*a8ok^mj+CO{&vznM@EaIvbcYx$ugm^i>jo=0$8xXP}p z0Qd3-_DO-h+!_>vJ1hgJ3=Hq%AMJsVI2icqDyg9?vXM60iPBc5xVB0+BL;X5A1Twq z73ccsJR-NJSI=by^ITlKc%1d=wo6qdsd&U{8$HCMPb@~fs1PYG+95JMN%K*eM|>Ir z4!W7OL?VS}lk>w7hv1E8Cnm<1CAwgml|U&-P`|;rgvVyVGW=?CpZCe2zI0oxTY!ZK zj)o=?FD4_3ODalUTj9S?Et4=|79o4_VdzE$Uoilvv zOdlL%%7+I>NgHOat7Gq;Zep<+L_)zs0e5xdpkr3v!m7ix6ZL771Q1FPdmxVz#E#iw zWuZ==n(8cn-QJ-b)=F8Ub+sVY{eRYJ((WZaTXRUA`7ohzgV4oUVccZH#!a9#2j4-c zA%8UxxrpOfYlb$e8O$AV#c9SPhc)AfY@5SMM^_(cRg)Vr*!z&_`Th>siC(;>Kg8H5 zUzUiX87Gdg&>dDNFsIr_*%|c!s~NX1RSz1jHPKW=H+l&yM;rnqMUEQFnhoLw;ZC~#TpS;9om6$jCm8~N{z8@3en#*DBAwGGl?NnmGOxS7{_@q zrkf|2+cAzWas*89L$x+3GSph1+EIV@1$-CfA=dri{v5mnl#*G!r*5qjlalS7u4LP2 zp6F((s6A{pP9cY|emuz_CXuk3Pa~FYV%QWt32H`s^JR38EC_2(j7UuMM6#}@{JF@% z)UGcf#D$Bt9ei;Fj13XO29ByWBgv(M#?d(btL=w8;2?>3`T0Iou7-JX2J!`^WHwgc zOXuI)TUTk^lC_|u1X$t(2n;CkO%Hal+knobO{~G=7c%-}> zVHtNZH(28iXi#h(RbbI%_mHL7C4uYM370nP?1ide7c6i9b>QWPxOUM3=jD+46(MlA z-UXfI-j@1aes79Wg^vpB%lD@~!kX*(0(R708$<5u>V5hu?&L@DwRLX{yLay9_r^ci zQ$N_<-I(Nu>fYMud)T~lW&FJl^2K6dcQU`WyS=)-TinC8pS$Zj?~ko*Y!^PtO>P%< z)%~rH_BSSW3u|gIpWhp~SJ++aQ?2W3ZWz=aYv_>^Vs$mMig6Ek!eRU9Z2E6fmd|&U zZpBg+3wWLwmGu2)4W<;F4Q~|h#|t*s$6!_cR+Lrk9!WH}Fn5PH%24>V%lV$NEaXJc z6Us6;5x!f+;O&!S%R6s)cXWpR;c-3tYPmqe%r8Tt$c(=HTvwl`)%vk?{dgMH^A4fQd@6V zleYCQbZzT>%Vu}=bU8y7-7Ia-sz4wE{goA$3$E7Xs%T89 zOV+JNH|n!!%)z(8AHy@saj}ir^-@h?`8lJkQVfdiCuZ&XDs|=9fezb5&^f-)rgJz# zF5pYrVUwBbN6khV;hAHlD%RLO=TK<9q}GOZ4(p|K?Ks1VN=gqgViGXdvA-(zuzf3q zxSeAx5RRoEPE)iDGX;-Cj+qOwz^dU93+y;JVq(Wd&oA$jhzjSyiUvr43oIx zgUyF|YDm%L9Xw%DxFaUi!}pl-HA0M-#5S`fy@m*paTXtDl&7&GIhKb%RVf~70_!+$ z?e7(5Z|bEbwzf0IOY|~}6`MvDU*nm^#_aol`FjT2b$V2>c$Jsk=RVY%#mXZcQwfX;aTtC=3OY)6WClstEajZdV3*; zKPmMTlK45k>kw0|un7%wioeA3>R1S)WVF~ee=MI#Z8wt46M;rDn#H0a0xd8jHh*NTNx-4W+UYW)Q?vtYDVIz;Egkp;Sm}UGhG0_1;{}Z{P6MK=+u?;$=7Lqy z;42Fe=3-USX=tL3)p@3(tBam_rZTEuwX-zz2pbWt#)y`9gzYVE1}?+aHflOsA$LP3 zZPQ~8VvLySrhTZ6CGR*JNiB@C*^7#Xndcd7>!u(EnD?nBUEuXSVbakWT@L9YWRxqE zBCNfqhF?Q3UqSq;Gk(>R3bN08`Bl@>8yZ%rXkfs`OR$)!9mJ!0K%&)|NtQ&SMY>yD zntiGn7I=YAm|<7E3`-)`xN0cuey$FZE97bC;90mjOU%_-Y9!?#tz`jMCzMG{!cVPR z6gy%iLJl1~khvw$=2ZLy95d^oJzk^uz(hpXKx_JZxhv4XunNGm>#jXMx17`%y<6 zaYVREZjm>g(6mKYhOrbQSU2Wa3MG*9i z8Ce;6{Rl9!q7fr2iI)w0bq)KPz~`@}itFD;g9^-;W5EEiK{0eczpSfhQrAonLD}WK zJz*vescW(kyl(*=vtB*}SAV@sl&hDM>X@%DH)&9`td(hOZc{6pMmZVn>SaIe>gBE~ zW_TT=ZgTEVnuZyCZBLkS{b-o|iMgjmo;A)OqAlGh5xE+rq~v;qYnWSz_A!%cmyMF2 zR*h0uk7Sn-?QXmZ=$RL;P71ppD;Iv9BK=3td!9dpS@5MjVQPJYi3ZR4w*18ptuy3OFWPa7@B@V>}5v{IL%nZ$4fFuA8jt#=Q$C+Xa#=_zs^i z$!PCy$N!()I~k`jKIOJsYxsE%R=S0G4wl+VyanJ_~j6n5m1$iPgm~a>3?X z&U&ZTY&)yMzYmeufO`aE0?j&^ZSB(pW8B99jq(^8WfzgxMWb}7)J^8OXIYAI@zYGX z;Ol+DlsjjcH!w3Jgk_)p%DxusJTw|KCN`}5L==+|-$|{!TdPK+VOUkKu%pE~Sp;Rr z!mN?V02Hxgr?@dmKYvd5$=WfRcsDk{7k!F?`JWoLoEM?ISIp0wBrTQJUhEc zd-^EW-N@cm*|fK$O}YDI)m2vKg!Arz7uJCmcK=n?t|XH|{G6K?RP+AH^!aYvyt}r# zIBRd$oLZd4&W>GnZ01*0?46gP)?y9bGWJ}jjUW^5Xv{9lPN`q+x>fc1+SV#+ zN}~X%A$y~1x4sp(P+oAiP_`fWJa#>vwf726a+85~CY~GQZUb*7?CgV_PL{{-Qw53&#qliKXl*7Z{9=a z9@t*jHDTDo2LabX{F(=$?W@`eXH{eSsx2|AdLw35HJ6%w3i*&mMO#z7$SOCYE8l?H zM;Wo}7XHe>>lXTqf2K%HQ^%|Fy_s~Z^pAsz+3d6;Y=bGRfxwnNP#cp-@+A+PR4HBs z>&t4u{=H{mHHcy|;ybCT_1rqIhSzF{il-Gufw+^$Ob|K2mHoqGt*(uJ@#gtO8XE$T z;7bWn}j5`{6uc{o?{+Qwh)x^~kBD=VtS zi^<{qhSb)58E)Lm>*VJQjn2=`xQm>zPT$q=10vxNi=QX;*y;gphE-B^Dw zwZqnrixeATO*8j~M5IvWBW0uug>u=2Mc!OQfXjt4;=yl3Q9Fl-K zeF}!Y+IB-f`B;ZKH2h$hTT;eNCT-h)YD+&!eZ(u2+F}l5Y}qV!h{bA{RM9A5A)2&% zb5A>WGufsUtoskii(Gbb*4NmXya@P#KD7jJA?ot}o;U#}Ni)Kb3wcgYM`&oG*QA*n zp|yCrj8^waqrO~64cISO7i?Jh(5R)Un2abcDSKXD#*f7o!znDZmD;Q7sV%)-#Ws8( zPNakoZX|J6qK!Z2kuu?v4I&cc?&6}e+A?EQH~osyr{`L+7wHCy065gVFWL<>;k0e) z-$~(d%mD5ea;B@WG3N+PDc%p-m$86$VCsiw(3VjOHD!DyZOQ+^u70SSh-t!jD6Qa_ zg8GGp9Zf^G-s-ni3tNF#z+*h{jG${YzD>usdurKB={nzUa{&#w56&n@#^RplVz5Y~HD<%?nnYbj<3%YMc# z7>AniMek4I>yDiepl&@e>#LLOEce&?aZ*~f=CO7(yu-#WCu1j)_QP)$MNRl<)FE3< zH}%GS+N%_ObLV&d(bs`y|@rH{#?*kNg1)e%AX$CSLvgMJ7zUSdMh8po%SxK%lOf!rrkF+ z5Xm&OFOv|aau~vtNJ@>DgUWVMlo->u+cyp{fvUi4(~wy@duP$lgx9-7buwans_D@6 zWj%2^__kzpBn6iWj1qpuo%tx43XkEhK56lf`jr@j&=~}f+6~m%s=>}sbx$N|E<0Ex z>0H-g0Gm7ESW;qHq~Uslt4nSLSrlnFH@MuhtuzIQ#S|pQE?3TUOR-oWcA}>?wdB*L z@>S)l2?fShRrx>&jQu_Y2EG4eFD*o$zZsM!lo2aU{=vY~#J7$!V+{;p^GIzOZ4d0ylKWn2RR$4oRN8Ko#@zodsq{Lmy+RMNo9{1 zOLL84W?U3m#iGQIFo*dKJ?1da=pIEeJ2SJu+lWxY7(-l2T&TdgNb8z|5_mfywp_Gz z?Te&{lrepZmA~1SSUEA;A9L^FU=3c_{WqHMYO>B6m^wvz(@7~1@(T$HIE6>?z?Kef zHHZx1-0o&cZf3qGBbbxuLTM3_GKa~f%=ZW>^IdUL#s#h0y9*KICxYUHGGfKa-xye& z_>2=qkXPAA05+g7F_?6$V=D+L(A=}R%z%35<0X9 zd>RF1oFbEQ=i8RC(Fw)}pf99~ZG@YgV5>Q9;csfK7J{#%$O93I$vdiZXQECpue%j9 zmkW#}yHbRu++ns0Wltz*JE_MDLEHU$yn4L35OaPs$Q;UuWe&x0pTXAhj2KIf<-(Td zzanDw7=4c*mxV2M{M%lAStOnyI)+%a}pJG zOyr`m3wLwJC_=sH5XyIR;#I_{Aie(dwey`5m#-b@rlrCYZj{u*SFq!bnnx#!$rxRx z#%<+S?$WHeq|@Noq>og-u4CO61O>0qEIOLrMrAn@&}d#7?H+*%TW2);Ep`UZ0(z~X z=g49HTh!Dz%$f_#YSCfkAdBYk)ANc+c=<-9Q`5|S%6-cQVTI3|qgiZho*!o0sfXAU zscGI?nz=oby)!pIn_JAyVIG9H^>xgiYd_) zVOS2;W?zoWe$pH0)eXQOl{- zbeaMy+Qi)a6RdQy*@rWBi#WfDhlAYW(=+Apt3D^m)1ldZx6k{yT2)#J?ZRSJc~{tl z_hanBurej={+kg>RY>3c>p^{&GGg`JUpqg2w};_)w%YG6#A&}rntB2o&1~pHZT)tn zW$a=y$2w^Vne0rCO&8A2Bvp1_`G9JK(6ZE8t$BbDO?G^uokqIq5mDxO5Ylh`Ic%v1 z7PY`Wbhe}Y$R|o!!7=@@L|K`l6qbt=sh21f8Zw*ntjpKaMIVYiOF9qb-TvmG^aVbD zPWIrVy7Edz`HMkPP(~~%es*9|OijX1qHA?*MUIV(&5GVuos1hij>pZgro}m|9-*5P zhi0>`w#yH%;JqBgNS z9VYwe!_4cV!ZIi<-NcG_tguXOCO4PPX4BcljL1n$m+UHL4`BT}c%M?+ExnN{;u!Yf z4!Z-kNL{7-rlDkDxeKWAdSsCEbBE!_B{Fr(f26$Pjh$drVIKhjHQ*uRsry?GW06UJ zWu;r*;AH)xPoga+muNeENwj25fIq5-v==?O{*`HJ`>bVxwC!=sEI;yOKpJ@E}!SkL(~_;k$#ea1hF5vq(t2mnF2AQs{nJX(;o6}ueXQ_SSj z+4;0qYgM)MOa`&nb^RUYJF_xm%=5J4kQN@I(@f$%<=eJJ^|3g6%SzIRT;+X;U2W<< z0$GogmqgFeldV2aZQW=*dFmlS2ZWg2yC zbVNU9J&U4@VKeA@vq3qBuA<=l=vhU%bfkNKFv^TJrv$nk|5KDvvs2S8I_2rDoT5f? ze=O3Cr4`%GuN4IZgF{BE=$^d5EfnQhjs3(^DuagZ3wSKJ+?`*G%7`y75KPlyA8t?U zrrZ;ryEt_pK}E6Ic?+%Fi^Vv2s#&`4c}%?oF}_}QYNRMwm7{TpO2UsxxlY}wl0x}% zE8>#{(aXz)Lg;?-nS@3O0!*92D5*yqCFej({%$<|{67pTXOt1Eoc)u5m9v+8Ghk?X z<2V-S+C!HzZTIPPFiHZznm93zuy;dSo5Wje>kr%O<*8P)1s@`*7gMZ1+8|f{^M=(F zYw|8b3goHV%H2R_v2)?AYzJ|SaiSs`$T<2Ti%RXZNCNB`b$y&G zi`JB3ZGt>)WL?4JomR89VptGWr&D&7*ULc#aFlhKY>E$IUQ#ZxwF)m8it<4D+r9hs ziea66#Zt?9i)0<@G=wV1%gT7l%D@9eWoPd1=hT^OQk&^DXFgl8W0EvXt+@4!8XveB zekAIzyJSC7ZU_^Jk&!8S!+eB#{`r|Z&f?0EMwPJppQFVj-}dQinB@N;A=xp>e?^e} zUL@K3eor9=|74KClo88diijF)jO_*(jEE&HfX#-c$Bvt|PFrmqTp^VIl-lm~AE0f<)*FQ+gua~x(jr3}**({@n!S;wBh|;2{ zH6^HpI}@cmhwj1xu7q<_-AFYUS%@UFK9VdYpK`I*&j55R$RTq4*&sP6BbFRLH843Y zPhhoNj4iIOT+Wvt>>^gf#KfCPr0;bOF&AuN-D*O5>E-eD28&r3&BF|_CaUl)plMsv z#j1u-!42kLnBoUCIFz8V|4v_xy>Vg;-Pln^rV1MX^em|ukkUw0XO{EoeLK@`yy}kG ze1Jz(H^o&+S@EhPb?hQj>liI;;NGWk%s@50%)sFz<+@ipbxFZe(~c7FDyD%;T=JYD zW&V6pEfSp3gS0QrIQhL4xW+)?;T)#Cixvo#|>~6nXSl&~&@|%UR;$D7l zzj(R0zq7NM-&x;U-M+HAwz8~l7WVQh`MvyQ`cRA8`@749iS^C=T0#AAcZw%)s&JVn zi_7_q!c=}^LuIMrL}7D(qqx4kHL<+CxmnoS!!1}D>fVPth3Vq`{7%6kC_6|<(BJJ% z&?EAZoA5IpAT%A5My(Qh1kwf@FdvTwCoL~GjdC-da?_&Rj2ne(-wF#Y=Mt-jZ?eF6 zUgsqd-RQ=Tu;|7W{v=Vf{akMVh{`dSFef}?EDQ7ve7Gk<-1R!#9{{QV%3zr($@5NCfQ$XUvWL5+tmPFtN8O`cR? z+m|NGdbv|#M%@8ysw1|v+7;jRjs%Pd+6o$A^yGf4>l+Dqr#25&SaIq})2t1#2!H4W zrkjUpl$&PdX0#3lATWWeR5etpI|b~YY4IE?*>^!8L=$-?+0d)yKv5!)->Q* z9)G${b(YPup$ZBYQ;FK(Z&>g#{5ZD9>3JltIGoe;#rE@cbH$ z?+D2v9|dfHh%J3y%4VpzBFObP4k?V_Wbg>1_p8dEOR7zls){D0k=nmB8t8?F_(aU` zX_Ax?%cp-iFrUV4sWi@}XJIUdeX~&~dwHIL<;-xL!_t`*qccQ&eeaGgEz0-mdm`(+ zId_jgS&(!6a!u5-AFhc@-Hmz!6TV!*6i9t4NGP5@?hQY264GwG%!POber^^K&sNWM zkBE+(m`)Hsn{@mFA~Bj+f0qsKI>hZde{y)O@m~!kK@UvrV`GSgT;j;giub)~mP1rd zJQO?Azxcez?h?i1Z;z3h926pQe^;_aju=zp`N=M0Qnjbf32V(GF88a-pG_(*52}hP z#O0R<6_*@{nAaEL>DPk7k}_h2Z63oUS_8o>u_VpmiC?l3+|9W7Oje=xT@P+7QOdV?LaeRws*Kk@W z;wUUst@$!9~<5|SH*`z+lB%I^78JKhS&*Qx;0>ekjbKU{IzzBeQ=!!d* z3Y=iWe{SB867lKq&vNEYmnn<_)0uUO2&TXfH+*CiizT)6s~X1F5x=Nb zb)!;kU*`h`20WvuFeUZFX+)M_=nPT1?T&Sm%^RWNhBl2ve~kF_n;3DY2osFQs}_%D zdMTX6R$I7!+^Jz$PP2zEtW*`Z#86qq)OA>;h``0LKa0=AP`FlZo=l1Lz7fB5!%eC@ zrtF$0hvI3Uw7ft2(M7*w@Qg(|y%w(7PLy?xL3c&|O@5w*IZBgUjO4yj&Jp zzSsA}xlGwE znD!fA7}WR3Z{vG4SZ}Ez_yYAPBbHmgKQOmQab(I4G5DmA{bNBB!JBrt_AB&4$*-ia(HM}92^>mnB{6S-0WQbb{%r0T~zw~wGnX)pj zEALg5!QA z{GSHa8EKPcF7s$)-HjPwdgSxWF+!2$DmGJSSm5CdI}ddP zw|30R`H|JRh3|zrxdW-5d{eBx0$ewoJ4NyJ%{ab#O87+S|LOo^U7nsxa`t~2inGpM(bKdpTBm_IA@E;L)3Td1 zo<|2!ZkBwogNfCS2Zdps23@w0in@4!vF41ODF=Ew&OFu6+ zL{oCC_}np+5lhqmZD5+xNW&FAv;etT!Qv~fpdI63cEc;W!_5n^H)K9HLfSX*zA!s) zAT2k+ypWxkjkn|Gdf4?O683woJNxR)B;?aEIm8Ad_Cz}jVx7bWiETspk2{kU}am24@Evyf^}hx>*< zRLsi@kt%`v=1kl+BOuk27FM+7uXc2fP8s_K-j`wL4aB7c!m219E{a8Q0?4jsKf=ns zHge6w%g&;1pE4Ag7&o5j>}R55sf* z`*@|)ekOO4g}aM6Blg)^iBE^3YCm+4;sl!S}~YlYqSur^2Sv;wb6M{qUGDNwH z4s>%U7v`6%%Q_?+m~z88eXJ856F$$6bF6qD=~#|W$N9orsY3&%f!RD!8=X3KbxysL zYMUMX%_HpZ!e;&JigKv9PM=t3Pm$dvigt0%9u7)B&K`axBVIRM_0!`3;tv}0B14qB zXa~>)bE!gO6`P1X$0jJ1XhSJy6R+#uJhDTFr{Hk&e~%5ljTnV<6KrBo! zQU87ZWKlGXn@1sPNj&WH(zA_84NaVQA`7tg`T9cKyXfV3&V0G6-;A;FOv9=+v`Vd6 zu4?L$Wm;NE#dab?@W@+YhL@N|iJ5Bipk`!dvMSa))hgPtkL_5+_YkCuV4YsOADYf2!x2(=K!AccRF*CXz#hRCoyw&tPIPoO%xyew?w(fb94$|Y?KH7okCQBGQ;?Z$6^q)nbtkE2r}!os z*ucYdk{j+CIJC(cwv#bWRodt&emd~4_U1lXie{?(6N?66*s?u@ABUBbVV`Ncw7_9f z!hd#?YRI?LWxU63Ikc^4>W%#FO5_Uu&+{kZpLR`%)?H6LQeO40h~?R65yYlHan?i8 zNk&ocg8rUgVFw)kD^H$7U~G3eQpJv{o_!cEnR?B0$Tr4vj(J&B2d4=10h9Ei(sj)` zq6@uef`yNo&G6=z;<>2z7{xxaHFuv%)Q((_f!iy}!gDtXcJ7iMT+QdjJUa#|BS8Qs}fj>jd^pY*W{EsbZC% z*UYlBEuDRjDXgxm)tVIyOigs!b+!!PIJUGeW2fb-u%1qqnumwz!OMm@%?`1CdmZB= z&$9!gQQ~_EqgSic^y@6s>Y$V3SKV|nXtztIp!1$bZ&{VeifNRWXuSGr8y>7B5^sEM zL3UWGH9KVpAgG?tc{8wH5p9GSuLk{LZ#4)zBR6RM5iE3gte>ilN*PL;ApkIS$xJJbjLPOuN6jMzTt4+idorY~V@ z@R25BPA-csh&qy&MOr)74D6Oy(+|yNold+MJkf4a_Ef2g-asdn@|t70)YuN5d>3f1 z7Bg)(f4bapCdQNdU-Tw-ODh?NMhOEOSV6ePHx6{h;2*RGr?;LEmeL z!T&hOV9JPP@b3-G;Oo5LS%1u1Pt3xW?N&!uPj$_lB<}-DP4UfYPt^ySlhhkG zn2W+&%!^ot{}>x^!gJxxeva+-u2yJ(pizfNR!il{R?|SD)~MJit$npwqrGYD_?Ky? zQfP*r1HCb_R@mN*r~W@pMEzqOT@lkfaJw_(y<2L4^zMn={LIXZsyC8~@>>q)-*>8> z+SFR9H?o=B{97nA(gpIGzZFBr^E^8-zUKvxlo!1&kpP3w@G#UFczr58&4z-nGuRkl zp%_d*j9)0m4`arKvG121>InXuU`Ie1u^qvqfjfd1$C_qEZ8XR0x>e1I@%~rU(j=+7 zydU_ubEF-0aLoh_Ak#$O(XwW!CHC1E!&*XUy`!aS2U+yl7^6>QS?ntMY>d&T-k5;I zH+B5hPBbP(j@}rzs!iOcB)%IH{J3wr$Bbma!ojRDRqqH9=PKJnke!v8(K~4_-`z4Z z2eguUrvYbdCcl!D+TWgs$y4go*y8Bn&48&RO%?W|MEYld9e0a&I06FSHymLf+rts|@hMDka<8#t`icHR?C zg7ruz0Y|m}mrb>Qq9YO%H_nFY9d$V>{DnW-Qb|)yl7N@Raor$6&D!|VSG4D8iC`1M zNnA)`prP6{M#|onbQ>mcSf0lUWc%>#fx)`%D~@zfE^CX z8Hx9pj(^K?_sS*WBkU7E-j-vMs{ycb9HZ;WQaPA}JrXUa7k0T^ZXJ=;T zc^mZ)5{%+qsxApJ3P2gB4l5}`dlttX802e)sVbw%h>sLXNEA8vL;RJg9;zFQTAkB z?zG4$rII_`lMQcq1I{iT4q>q=MMp^3?z!72+X%6IJXY%zg@+$MhDTa!w4GlA5A_l!Jxi~4NW^&{)v5){8jkyc0eI+9b&^Kd-PinX@YrnFeA|e=>+XAOoKI^ zJP|1B$r%?txs&BK1eD#6(hoJ?T-@}AT{|{7^U7_^vL@NDmUzw<9=TGAi`hO=V z^eH1&=>PV>LjOi+3~(GWghmI)F{=h*+Zts}pN93Y9*MB~U!x%{PmecoqGMudTet9| z68z)Vm|ll(-x_OGu?v_rihXz+nv#^zrZj+sD~ApWD?dpjYY*_Ucpjw8qh!lf}+~ z$+9Pxp&COoZnVcV{bL$Lw@19ll+z|2pN>GGRMyv??+XIWR=V_3cY*!Y+!lacg!=|{dPO7>s&V(QvPJl7d zb?M}&eT|DnHt7u#uaSVQ|7UNu9zzAiP*@XDFe?0OX47gOwp9v4RS!Bwt&F|6hH!OJ z=^0*n+9{od+oqk;GpubuM63#2ewa1m>P`SZXV;cm%gDOUPl=|Bfoaf?jSS~^rw=FXAN#} zA)cF&;`w~qOQ`GkAA?;7WyE$Je>8B{F?HFF7J;OlJUA4a$9go|tIV?nzb4$+(2GZ{ z9Io`bm&U=8s%BLk=Cf`nUReL%1JU2ApLZ8v)i10YSO|6l>Fwg?;9bER5nVwDGwKfd zzW7_OO9(A!{WD3s1T1LX5nY0nv`aXH_7mzB{xH}rP)2OG@bAAByM=$>yIVNHLb1(* zX4}FnAPqZknw=8)r-#r4oDwrAG3}P6t@)COtP> z%BB|=7t86xnZ<0jq|eRHF3hJ(xbpVLR=b>3L73Suri#wPr6@iywXGe&eO<2TZd~ey zcF@E?Bvwg{QV(8s9xnhG^#PG5A1R-U*xsTR&#<%zP?rp5BjomM-uif99D1Pr|VXv@T94mhBYN4<wEhvg^7*)9!pHE zZ|2tu>Q;WUAd>W>srU06DsFp!du4pLu$kX|uQ;;5zrMnBB3o06*Ny(TcPDt-?7)k7 zs8%%-(KC7lv&u>|oP&k$Iz#k?qS!PmHbcdx-C{EvIn9a1UH&jrSlcYB$Wv$T-chk; z&Aht)K~{bLeMIGE7b!G%R!zJk&Kph)z?um!QaQERq$zEwWtzCadc9+vjpar4cAxEu z40UV$g{3XUHCs(Hx&873?udtd5JvXwIn4P5)E<}%69NAjDgqx6AEPLsYHe5 z5&e~t7;_0TaNY*bikT*oW}_(~@lSlPdd#8_3wytwu>|58sX*4?=gOncNv zTgF4ZHe%K-{U#Rtz~c27*()_+a8XRh^IE-yC3R7@S~e^w8#F|EEihg2G9r~qM||oc zhP*A6ZjQQGh>6DC$0*flU8y6?525rJ6UT`74c`$=y7IJELRgc|A}kx%y$5vmvV}sM z7(uZk8uzXT+)58F6Yh zYoE$4ifeMBoY`4%@tj?aIlCHj_TA^~yU*L#%-h$@XGK-#?ThE_ix=#wF4$FF$cp+b z*i~J~iTd2JFTP`6e8;}{j(zbR`{KJZ0+zd30n6R2!0>KPV0bqtDz-Qyo_%pf+-or_ zo_#SVo_#ULE0&!R7bC$h&d!L7v)D3`L5GC67%85Uo#FS&X0yCv*=&wi3<+K_WQdD% z_Qg5-;+%bPj$NFaL6e|+u>{?V66_7K1XYX@>`I8(|3n4{syL;cbK^=v7dHwGoyOPr=&=a(Ek| z1bgk2V6UAec*R5;0d;vBp@g_tv=Pc-Z3GG4Mp%O1OSBQnVXvJfc*R5;K@M*tlwfUy z686QSjo_ZVjZlJB3<=&wD8WELg1<^6cpE`&7=v?EQ)-Va!77Fv{&uoxB;wR8YpS?r zmRAD_UJXjHI%m-~SW~5hsK%UKjXAp-b9Oc6?ThE_i|6f&=k1HkHzZ1k%HQP;2|2vgWzi&v)3|t%bv7&^ zo_&!wB%ETMO%88cTnxmi1E3t%Gf_fZj06W9S=2M(6bcXt-jI;PTOB2M#n2?g#X0-p zoPBYQUyN45UmOzreNnq%RmEL6;B-57FBHcMM1q%t1ix1fuaeae3I4hvbnKGn?UEPl z0vGH8@9>w4JMnjl1b>xi&HU{k!Cwjz{9WK-_$xqyBM*G}+d+cA6eRe&U`P>ANN_OG zh&kX$aGa6g@S>;VKq0~LL9^yhfdqdFBsf06g1;{$c*(N>fdP&L#|H_H&n(bj7$RYp zykM8SV3)jLmkh@%)fuvwg9%b`ppf7|A;E!~MPp;{3kePs5*#Shk2h>2czd2jvmrT_ zLxKZ^1P5vsPs~_}L~eEjBVP4*C(dsrapKs~&{##Ph2SczSuuEEieLA&OU61!Eerl_ zt;9AFOYWVy=?_l9+FHv|ewdWs4`WxseK>N(#9s7B`Mfu(ggIH5CzzRz?9g=}vlKhw z=5@QocJq2mc7M^kfWTR=5p)rQ;KCo()4W>u`KYnGK&GdO%_(kmT$fhvSsxa)BqGhJ+1EYMsd0^=#Uspw?DECQc znU&{Hj7Ax@4?^PEFWLGmpXrvVVh578^}KmF9mb$Ck*D7 z^%G2($3*(qBhT;>zb*Q_DEo6-yWX@~FpG_n3i}gtd|=9|n9+{~0Zje#y-)MId`HYh z;#STG37>~VQ7B3!j+PJU*^;kMzN14KFBT7nErLXsd+iW(}?b zK6g+(!Pd?y$xM3`G(%7F4wR|I411h+vRou3UikjX)`!cfWjsXc9-bmf8eI!KEeW4s zo;wGS2?Hi(x{W%rvrG;IKG+GbL1KaK&;PsYlL+5k zRT2kCVO4MWz6k7!ZJ)?>Jy;ZF} zfptGw;wH-@9nO(3*ZoEzaazBR zgqYc8G$Uarl3mv~<$Va7>Bt{b1UJYpiGV~vA|Mfv2uK7Z0ulj str | None: + """OGC URI of a CRS (``None`` for CRS definitions without authority code).""" + if isinstance(srid, int): + return f"http://www.opengis.net/def/crs/EPSG/0/{srid}" + + try: + authority = CRS.from_user_input(srid).to_authority(min_confidence=100) + except CRSError: + return None + + if authority is None: + return None + + auth, code = authority + return f"http://www.opengis.net/def/crs/{auth}/0/{code}" def _quote_ident(name: str) -> str: @@ -70,7 +94,7 @@ def _s_intersects_bbox( south: float, east: float, north: float, - srid: int | None = None, + srid: Srid | None = None, *, tag: GeometryLiteralWrapper, ) -> Expr: @@ -113,7 +137,7 @@ def _s_intersects_bbox( def _transform_cql_geom_literals( node: Any, - target_srid: int, + target_srid: Srid, reproject: GeometryLiteralWrapper, ) -> Any: """Walk a user-supplied cql2-json tree and wrap every GeoJSON geometry @@ -180,7 +204,8 @@ class Column(BaseModel): type: str description: str | None = None geometry_type: str | None = None - srid: int | None = None + # EPSG code, or CRS definition (see `Srid`) + srid: Srid | None = None bounds: list[float] | None = None mindt: str | None = None maxdt: str | None = None @@ -319,8 +344,8 @@ def dt_bounds(self) -> list[str] | None: @property def crs(self): """Return crs of set geometry column.""" - if self.geometry_column: - return f"http://www.opengis.net/def/crs/EPSG/0/{self.geometry_column.srid}" + if self.geometry_column and self.geometry_column.srid is not None: + return crs_uri(self.geometry_column.srid) @cached_property def geometry_columns(self) -> list[Column]: @@ -401,14 +426,14 @@ def queryables(self) -> dict: return {**geoms, **props} @abc.abstractmethod - def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Tag a cql2-json geometry literal, whose coordinates are already in ``srid``, with that CRS (returns a cql2-json function call node). """ ... @abc.abstractmethod - def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Reproject a EPSG:4326 cql2-json geometry literal to ``srid`` (returns a cql2-json function call node). """ diff --git a/tipg/duck/collections.py b/tipg/duck/collections.py index 8b172bef..6cba9f9b 100644 --- a/tipg/duck/collections.py +++ b/tipg/duck/collections.py @@ -1,8 +1,10 @@ """tipg DuckDB collection models.""" +import functools import math import re from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from typing import Any import duckdb @@ -10,9 +12,11 @@ from cql2 import Expr from morecantile import Tile, TileMatrixSet from pydantic import ConfigDict, Field +from pyproj import CRS +from pyproj.exceptions import CRSError from starlette.concurrency import run_in_threadpool -from ..collections import Collection, Column, Feature, ItemList, _quote_ident +from ..collections import Collection, Column, Feature, ItemList, Srid, _quote_ident from ..errors import InvalidGeometryColumnName, InvalidLimit, InvalidPropertyName from ..logger import logger from ..settings import FeaturesSettings, MVTSettings, TableConfig, TableSettings @@ -83,7 +87,7 @@ def _coords_bounds(coords: Any) -> list[float] | None: ] -def _literal_bounds(literal: Any, srid: int) -> list[float] | None: +def _literal_bounds(literal: Any, srid: Srid) -> list[float] | None: """Bounds of a cql2-json geometry literal, in the column CRS (``srid``). Literals of columns not in EPSG:4326 are expected to be tagged with their @@ -125,11 +129,18 @@ def _quote_literal(value: str) -> str: return "'" + value.replace("'", "''") + "'" -def _crs(srid: int) -> str: - """DuckDB CRS identifier for an EPSG code.""" +def _crs(srid: Srid) -> str: + """DuckDB CRS identifier for an EPSG code (or a CRS definition, as is).""" + if isinstance(srid, str): + return srid return f"EPSG:{srid}" +def _crs_literal(srid: Srid) -> str: + """Quoted DuckDB CRS identifier (SQL string literal).""" + return _quote_literal(_crs(srid)) + + class DuckCollection(Collection): """Model for DuckDB Table and View.""" @@ -153,11 +164,11 @@ def _qualified_name(self) -> str: _quote_ident(p) for p in (self.database, self.dbschema, self.table) ) - def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Tag a geometry literal with its CRS (DuckDB ``ST_SetCRS``).""" return {"op": "st_setcrs", "args": [geom, _crs(srid)]} - def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Reproject a EPSG:4326 geometry literal to ``srid`` (DuckDB ``ST_Transform``).""" return {"op": "st_transform", "args": [geom, _crs(4326), _crs(srid)]} @@ -211,7 +222,7 @@ def _geom_expr( g = _quote_ident(geometry_column.name) if geometry_column.srid != 4326: - g = f"ST_Transform({g}, '{_crs(geometry_column.srid)}', '{_crs(4326)}')" # type: ignore [arg-type] + g = f"ST_Transform({g}, {_crs_literal(geometry_column.srid)}, {_crs_literal(4326)})" # type: ignore [arg-type] if bbox_only: g = f"ST_Envelope({g})" @@ -275,7 +286,7 @@ def _select_mvt( geom = ( f"ST_Intersection(" f"ST_MakeEnvelope({west}, {south}, {east}, {north}), " - f"ST_Transform({geom}, '{src_crs}', '{geo_crs}')" + f"ST_Transform({geom}, {_quote_literal(src_crs)}, {_quote_literal(geo_crs)})" f")" ) src_crs = geo_crs @@ -660,20 +671,46 @@ def _normalize_type(data_type: str) -> str: return _DUCK_TYPES.get(t, "text") -def _geometry_srid(data_type: str) -> int: - """SRID of a geometry column, from its type (e.g `GEOMETRY('EPSG:26915')`). +@functools.lru_cache(maxsize=256) +def _parse_crs(crs: str) -> Srid | None: + """EPSG code of a CRS definition when it is exactly an EPSG CRS, else the + definition itself (``None`` if it can't be parsed by pyproj). + """ + try: + authority = CRS.from_user_input(crs).to_authority(min_confidence=100) + except CRSError: + return None + + if authority: + auth, code = authority + if auth == "EPSG" and code.isdigit(): + return int(code) + if (auth, code) == ("OGC", "CRS84"): + return 4326 + + return crs + - Geometries without CRS (or in OGC:CRS84) are assumed to be in EPSG:4326. +def _geometry_srid(data_type: str) -> Srid: + """CRS of a geometry column, from its type (e.g `GEOMETRY('EPSG:26915')`). + + Returns the EPSG code (geometries without CRS, or in OGC:CRS84, are + assumed to be in EPSG:4326), or the CRS definition (e.g PROJJSON from + GeoParquet metadata, or `AUTH:CODE`) for CRS without EPSG code. """ if m := _GEOMETRY_CRS.match(data_type): - auth, _, code = m.group("crs").partition(":") + crs = m.group("crs") + auth, _, code = crs.partition(":") if auth.upper() == "EPSG" and code.isdigit(): return int(code) - if m.group("crs").upper() not in ("OGC:CRS84", "CRS84"): - logger.warning( - f"Unsupported geometry CRS {m.group('crs')}, assuming EPSG:4326" - ) + if crs.upper() in ("OGC:CRS84", "CRS84"): + return 4326 + + if (srid := _parse_crs(crs)) is not None: + return srid + + logger.warning(f"Unsupported geometry CRS {crs[:100]}, assuming EPSG:4326") return 4326 @@ -703,18 +740,78 @@ def _parquet_view_source(view_sql: str | None) -> tuple[str, bool] | None: def _parse_covering(meta: dict[str, Any]) -> dict[str, str] | None: - """SQL expressions of a GeoParquet bbox covering (e.g ``"bbox"."xmin"``).""" - covering = (meta.get("covering") or {}).get("bbox") or {} + """SQL expressions of a GeoParquet bbox covering (e.g ``"bbox"."xmin"``). + + Returns ``None`` without bbox covering, and an empty dict for a malformed + one (rejected by ``_valid_covering``). + """ + covering = (meta.get("covering") or {}).get("bbox") + if covering is None: + return None + + if not isinstance(covering, dict): + return {} + exprs = {} for key in _COVERING_KEYS: path = covering.get(key) - if not path or not all(isinstance(p, str) for p in path): - return None + if ( + not path + or not isinstance(path, list) + or not all(isinstance(p, str) for p in path) + ): + return {} exprs[key] = ".".join(_quote_ident(p) for p in path) return exprs +def _valid_covering( + cursor: duckdb.DuckDBPyConnection, + covering: dict[str, str], + column_types: dict[str, str], +) -> bool: + """Check that a bbox covering matches the actual column types. + + GeoParquet metadata can be wrong (e.g. a covering pointing to + ``bbox.xmin`` while the struct fields are ``min_x``...). The covering + expressions are bound against typed NULL values, so no data is read. + """ + if any(key not in covering for key in _COVERING_KEYS): + return False + + # only bind the columns the covering refers to (e.g. `"bbox"` for `"bbox"."xmin"`) + exprs = [covering[key] for key in _COVERING_KEYS] + referenced = { + name: column_types[name] for name in _covering_columns(covering, column_types) + } + if not referenced: + return False + + columns = ", ".join( + f"NULL::{dtype} AS {_quote_ident(name)}" for name, dtype in referenced.items() + ) + try: + cursor.execute(f"SELECT {', '.join(exprs)} FROM (SELECT {columns}) LIMIT 0") + except duckdb.Error: + return False + + return True + + +def _covering_columns( + covering: dict[str, str], + column_types: dict[str, str], +) -> list[str]: + """Names of the columns a bbox covering refers to.""" + exprs = [covering[key] for key in _COVERING_KEYS] + return [ + name + for name in column_types + if any(e == (q := _quote_ident(name)) or e.startswith(f"{q}.") for e in exprs) + ] + + def _parse_geo_metadata(geo: str) -> dict[str, dict[str, Any]]: """Bbox, geometry types and bbox covering per column, from a GeoParquet ``geo`` metadata value (each one is ``None`` when unknown). @@ -755,7 +852,10 @@ def _merge_geo_metadata(col: dict[str, Any], other: dict[str, Any]) -> None: else: col["geometry_types"] |= other["geometry_types"] - if col["covering"] != other["covering"]: + # a malformed covering (empty dict) in any file invalidates the source + if col["covering"] == {} or other["covering"] == {}: + col["covering"] = {} + elif col["covering"] != other["covering"]: col["covering"] = None @@ -805,6 +905,34 @@ def _geoparquet_metadata( return merged or {} +def _geoparquet_metadatas( + connection: duckdb.DuckDBPyConnection, + paths: set[str], + max_workers: int, +) -> dict[str, dict[str, dict[str, Any]]]: + """GeoParquet metadata of multiple parquet sources, read in parallel. + + Each worker uses its own cursor (DuckDB cursors can run queries + concurrently on the same database instance). + """ + + def _read(path: str) -> dict[str, dict[str, Any]]: + cursor = connection.cursor() + try: + return _geoparquet_metadata(cursor, path) + finally: + cursor.close() + + if not paths: + return {} + + paths_list = sorted(paths) + with ThreadPoolExecutor( + max_workers=max(1, min(max_workers, len(paths_list))) + ) as pool: + return dict(zip(paths_list, pool.map(_read, paths_list))) + + def _apply_geo_metadata( cursor: duckdb.DuckDBPyConnection, column: dict[str, Any], @@ -813,7 +941,7 @@ def _apply_geo_metadata( """Set a geometry column bounds (in EPSG:4326) and type from GeoParquet metadata.""" xmin, ymin, xmax, ymax = meta["bbox"] if column["srid"] != 4326: - env = f"ST_Transform(ST_MakeEnvelope($1, $2, $3, $4), '{_crs(column['srid'])}', '{_crs(4326)}')" + env = f"ST_Transform(ST_MakeEnvelope($1, $2, $3, $4), {_crs_literal(column['srid'])}, {_crs_literal(4326)})" xmin, ymin, xmax, ymax = cursor.execute( f"SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e) FROM (SELECT {env} AS e)", [xmin, ymin, xmax, ymax], @@ -884,7 +1012,10 @@ def _collection_index( # noqa: C901 ).fetchall() table_columns: dict[tuple[str, str, str], list[dict[str, Any]]] = {} + # raw DuckDB column types, used to validate GeoParquet bbox coverings + table_types: dict[tuple[str, str, str], dict[str, str]] = {} for db, schema, table, name, data_type, comment in columns_rows: + table_types.setdefault((db, schema, table), {})[name] = data_type col: dict[str, Any] = { "name": name, "type": _normalize_type(data_type), @@ -904,7 +1035,8 @@ def _collection_index( # noqa: C901 table_confs = table_settings.table_config fallback_key_names = table_settings.fallback_key_names - collections: dict[str, Collection] = {} + # Select the relations to expose + selected: dict[str, tuple[Any, ...]] = {} for db, schema, table, entity, comment, view_sql in relations: if schema not in settings.schemas: continue @@ -916,7 +1048,7 @@ def _collection_index( # noqa: C901 if settings.exclude_tables and table_id in settings.exclude_tables: continue - if table_id in collections: + if table_id in selected: logger.warning( f"Collection {table_id} already registered, skipping `{db}`.{table_id}" ) @@ -928,11 +1060,58 @@ def _collection_index( # noqa: C901 ): continue - parquet_source = _parquet_view_source(view_sql) + selected[table_id] = ( + db, + schema, + table, + entity, + comment, + columns, + _parquet_view_source(view_sql), + ) + + # Read the GeoParquet metadata of parquet views, in parallel (one + # footer request per file, which adds up for many remote files) + geo_metadatas: dict[str, dict[str, dict[str, Any]]] = {} + if settings.spatial_extent: + geo_metadatas = _geoparquet_metadatas( + connection, + {src[0] for *_, src in selected.values() if src}, + settings.metadata_threads, + ) - geo_metadata = {} - if settings.spatial_extent and parquet_source: - geo_metadata = _geoparquet_metadata(cursor, parquet_source[0]) + collections: dict[str, Collection] = {} + for table_id, ( + db, + schema, + table, + entity, + comment, + columns, + parquet_source, + ) in selected.items(): + geo_metadata = ( + geo_metadatas.get(parquet_source[0], {}) if parquet_source else {} + ) + # GeoParquet bbox coverings: sources with a covering which doesn't + # follow the spec (e.g. pointing to missing struct fields) are excluded + bbox_coverings = { + name: meta["covering"] + for name, meta in geo_metadata.items() + if meta["covering"] is not None + } + types = table_types.get((db, schema, table), {}) + if invalid := [ + name + for name, covering in bbox_coverings.items() + if not _valid_covering(cursor, covering, types) + ]: + logger.warning( + f"Collection {table_id} excluded: invalid GeoParquet bbox " + f"covering for column(s) {invalid} " + f"({ {n: bbox_coverings[n] for n in invalid} })" + ) + continue _geometry_metadata(cursor, columns, geo_metadata) @@ -988,11 +1167,7 @@ def _collection_index( # noqa: C901 id_column=id_column, datetime_column=datetime_column, geometry_column=geometry_column, - bbox_coverings={ - name: meta["covering"] - for name, meta in geo_metadata.items() - if meta["covering"] - }, + bbox_coverings=bbox_coverings, connection=connection, ) diff --git a/tipg/duck/settings.py b/tipg/duck/settings.py index ad97465d..4f8525b1 100644 --- a/tipg/duck/settings.py +++ b/tipg/duck/settings.py @@ -46,6 +46,8 @@ class DuckDBCatalogSettings(BaseSettings): never have a spatial extent. The GeoParquet bbox `covering` columns (if any) are also read from this metadata and used to speed up spatial filters (`bbox`, tiles, cql2 spatial predicates). + metadata_threads: number of parquet sources whose metadata is read + concurrently when building the catalog. """ @@ -53,6 +55,7 @@ class DuckDBCatalogSettings(BaseSettings): tables: list[str] | None = None exclude_tables: list[str] | None = None spatial_extent: bool = True + metadata_threads: int = 16 only_spatial_tables: bool = True model_config = { diff --git a/tipg/pg/collections.py b/tipg/pg/collections.py index 3004183c..6baefc38 100644 --- a/tipg/pg/collections.py +++ b/tipg/pg/collections.py @@ -9,7 +9,7 @@ from morecantile import Tile, TileMatrixSet from pydantic import ConfigDict, Field -from ..collections import Collection, Column, Feature, ItemList, _quote_ident +from ..collections import Collection, Column, Feature, ItemList, Srid, _quote_ident from ..errors import ( InvalidGeometryColumnName, InvalidLimit, @@ -63,11 +63,11 @@ def _qualified_name(self) -> str: """SQL-quoted ``"schema"."table"`` identifier for this collection.""" return f"{_quote_ident(self.dbschema)}.{_quote_ident(self.table)}" - def _tag_geometry_literal(self, geom: Any, srid: int) -> Any: + def _tag_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Tag a geometry literal with its SRID (PostGIS ``ST_SetSRID``).""" return {"op": "st_setsrid", "args": [geom, srid]} - def _reproject_geometry_literal(self, geom: Any, srid: int) -> Any: + def _reproject_geometry_literal(self, geom: Any, srid: Srid) -> Any: """Reproject a EPSG:4326 geometry literal to ``srid`` (PostGIS ``ST_Transform``).""" return {"op": "st_transform", "args": [geom, srid]} From 3c7d9577eac46e7624a530354b0941e6ad46829b Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 12 Nov 2026 16:42:35 +0100 Subject: [PATCH 19/22] perf: pin duckdb>=2.0 --- docs/src/advanced/duckdb.md | 94 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 3 +- uv.lock | 8 ++-- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index 65146693..1105372f 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -11,6 +11,8 @@ python -m pip install "tipg[duckdb]" ``` +The DuckDB backend requires `duckdb>=2.0` (CRS aware `GEOMETRY` type). + ### Settings `tipg.duck.DuckDBSettings` (connection) and `tipg.duck.DuckDBCatalogSettings` (catalog) can be set with `TIPG_DUCKDB_` prefixed environment variables. @@ -161,9 +163,99 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) - **Extents and geometry type**: tables and views are never scanned to build the catalog. The spatial extent and geometry type of parquet `sources` are read from their [GeoParquet](https://geoparquet.org) metadata (`bbox` and `geometry_types`, in the file footers). Other collections have no spatial extent and a generic `Geometry` type, and no collection has a temporal extent. - **Items order**: without `sortby`, features are returned in their insertion order (no default sort on the primary key). - **Feature ids**: single parquet file `sources` are read with `read_parquet(..., file_row_number = true)`, and the added `file_row_number` column (row position in the file, starting at 0) is used as id when the collection has no other id column (`TIPG_TABLE_CONFIG__..._pk` or fallback key names). Looking up a feature by id (`/items/{itemId}`) then only reads one row group. Other collections without primary key use the feature position in the response as `id`, and don't support `/items/{itemId}`. -- **Spatial filters**: DuckDB has no spatial index on parquet files. When a GeoParquet (1.1+) source declares a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column, spatial filters (`bbox`, tile envelope, and cql2 spatial predicates in the top-level `AND` of the filter) also filter the covering columns, so DuckDB can skip the row groups outside of the filter using their statistics. Without covering, spatial filters scan the whole geometry column. +- **Spatial filters**: DuckDB has no spatial index on parquet files. When a GeoParquet (1.1+) source declares a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column, spatial filters (`bbox`, tile envelope, and cql2 spatial predicates in the top-level `AND` of the filter) also filter the covering columns, so DuckDB can skip the row groups outside of the filter using their statistics. Without covering, spatial filters scan the whole geometry column. Parquet sources whose covering doesn't match the file (e.g. pointing to missing struct fields) are not added to the catalog (a warning is logged). See [Data layout and performance](#data-layout-and-performance). - **Parquet metadata cache**: parquet files metadata (footers) are cached in memory (`parquet_metadata_cache`), so they are only fetched once per file. - **`numberMatched`**: only returned for requests without filter (DuckDB answers `COUNT(*)` from metadata), counting the features matching a filter would need a full scan. - **Vector tiles**: `ST_AsMVT` only supports boolean, integer, float and string properties; `DECIMAL` values are cast to `DOUBLE` and other types (dates, timestamps, lists, ...) to `VARCHAR`. - **Functions**: SQL function layers are not supported. - **WKT output** (CSV, (nd)json): DuckDB's `ST_AsText` formatting (e.g `SRID=4326;POINT (1 2)`). + +### Data layout and performance + +DuckDB has no spatial index: for a `bbox` filter or a tile, it can only skip the data whose row group statistics show that it's outside of the requested area. How the data is laid out (sorted, grouped, with or without bbox columns) has a much bigger impact on performance than anything in `TiPg`'s configuration, especially for remote files where every row group read is an HTTP request. + +For example, a 2.3M polygons GeoParquet file (~870 MB) served from a remote URL: + +| Layout | Data read for one tile | Time | +| ------ | ---------------------- | ---- | +| Rows not sorted (every row group covers the whole dataset extent) | ~870 MB (whole file, 463 HTTP requests) | ~16s | +| Bbox filter on row groups which don't overlap the tile | ~0.6 MB (file footer only, 2 HTTP requests) | ~2s | + +Row groups can only be skipped when: + +1. **the rows are spatially sorted**: features close to each other are stored in the same row groups, so each row group covers a small area. With unsorted rows, every row group covers (almost) the whole dataset extent and overlaps every tile. +2. **DuckDB has statistics to compare with the filter**: a [bbox covering](https://geoparquet.org/releases/v1.1.0/#bbox-covering-encoding) column for GeoParquet files, or the geometry column statistics for DuckDB tables. + +#### Recommended: DuckDB tables + +For data served often (or from slow remote storage), copy it into a DuckDB database file, sorted along a [Hilbert curve](https://en.wikipedia.org/wiki/Hilbert_curve). DuckDB keeps bounding box statistics of `GEOMETRY` columns for each row group of a table, so no bbox column is needed: + +```sql +INSTALL spatial; +LOAD spatial; + +CREATE TABLE parcels_raw AS SELECT * FROM read_parquet('https://example.com/parcels.parquet'); + +CREATE TABLE parcels AS +SELECT * FROM parcels_raw +ORDER BY ST_Hilbert(geometry, (SELECT ST_Extent(ST_Extent_Agg(geometry)) FROM parcels_raw)); + +DROP TABLE parcels_raw; +``` + +```bash +TIPG_DUCKDB_DATABASE=parcels.duckdb uvicorn app:app --port 8000 +``` + +!!! note + + Tables are not scanned to build the catalog: collections from DuckDB tables have no spatial extent (see [Differences with the PostgreSQL backend](#differences-with-the-postgresql-backend)). + +#### GeoParquet files + +For parquet `sources`, write [GeoParquet 1.1](https://geoparquet.org/releases/v1.1.0/) files with: + +- **spatially sorted rows** (Hilbert curve, or sorted by bounding box) +- **a bbox covering column**, which follows the specification (a struct column with `xmin`, `ymin`, `xmax` and `ymax` fields, declared in the `covering` of the `geo` metadata). Sources with a covering which doesn't match the file are not served. +- **row groups of 50 000 to 100 000 rows**: smaller row groups mean more HTTP requests, bigger ones more data read outside of the requested area. + +With DuckDB (Hilbert sort) and [GDAL](https://gdal.org/en/stable/drivers/vector/parquet.html) (>= 3.9, bbox covering): + +```sql +-- DuckDB +COPY ( + SELECT * FROM 'parcels.parquet' + ORDER BY ST_Hilbert(geometry, (SELECT ST_Extent(ST_Extent_Agg(geometry)) FROM 'parcels.parquet')) +) TO 'parcels_hilbert.parquet' (FORMAT parquet); +``` + +```bash +# GDAL keeps the rows order +ogr2ogr -f Parquet parcels_sorted.parquet parcels_hilbert.parquet \ + -lco WRITE_COVERING_BBOX=YES \ + -lco ROW_GROUP_SIZE=65536 +``` + +!!! note + + GDAL can also sort the features with `-lco SORT_BY_BBOX=YES`, but in our tests the Hilbert curve sort gave a better spatial grouping: for a tile covering ~1% of a dataset of 1M random points (16 row groups), 1 row group was read with the Hilbert sort and 5 with `SORT_BY_BBOX`. + +With [GeoPandas](https://geopandas.org/en/stable/docs/reference/api/geopandas.GeoDataFrame.to_parquet.html): + +```python +import geopandas + +gdf = geopandas.read_parquet("parcels.parquet") +gdf = gdf.iloc[gdf.hilbert_distance().argsort()] +gdf.to_parquet("parcels_sorted.parquet", write_covering_bbox=True, row_group_size=65536) +``` + +!!! warning + + DuckDB's `COPY ... TO` doesn't write the bbox `covering` metadata, and adding it with `KV_METADATA` results in two `geo` metadata entries in the file. Use DuckDB to sort the data, but GDAL or GeoPandas to write the final GeoParquet file with a bbox covering. + + DuckDB can write the native Parquet `GEOMETRY` type, with bounding box statistics (`GEOPARQUET_VERSION 'V2'`), but those statistics are not used when the file is read through a view, which is how `TiPg` registers parquet `sources`. + +#### Low zoom levels + +A good layout only helps when a tile covers a small part of the dataset: a tile at zoom level 5 over the whole dataset still reads every row group, and only returns the first `TIPG_MAX_FEATURES_PER_TILE` features. For large datasets, only request tiles at higher zoom levels (e.g with the `minZoom` option of your map library's source). diff --git a/pyproject.toml b/pyproject.toml index ab7b968c..177633f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,8 @@ metrics = [ "prometheus-fastapi-instrumentator>=8.0.2", ] duckdb = [ - "duckdb>=1.4", + # >=2.0: CRS aware GEOMETRY type, `ST_SetCRS`, `geometry_always_xy` + "duckdb>=2.0", # required by duckdb to return TIMESTAMP WITH TIME ZONE values to Python "pytz", ] diff --git a/uv.lock b/uv.lock index 8e087022..8828dc86 100644 --- a/uv.lock +++ b/uv.lock @@ -3,8 +3,8 @@ revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12'", "python_full_version < '3.11.5'", @@ -2301,8 +2301,8 @@ version = "2.5.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", ] sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } @@ -3720,8 +3720,8 @@ version = "4.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12'", ] @@ -3859,7 +3859,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.23.0" }, { name = "ciso8601", specifier = "~=2.3" }, { name = "cql2", specifier = ">=0.5,<0.7" }, - { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.4" }, + { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=2.0" }, { name = "fastapi", specifier = ">=0.100.0" }, { name = "geojson-pydantic", specifier = ">=1.0,<3.0" }, { name = "jinja2", specifier = ">=2.11.2,<4.0.0" }, From 2c21850835b5c2ccb212c9f83697e4c47159645c Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 13 Nov 2026 11:20:44 +0100 Subject: [PATCH 20/22] chore: make duckdb experimental --- docs/src/advanced/duckdb.md | 25 ++++++++++++++++++++++++- examples/duckdb/.env-example | 3 +++ tipg/duck/__init__.py | 10 ++++++++++ tipg/errors.py | 4 ++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 examples/duckdb/.env-example diff --git a/docs/src/advanced/duckdb.md b/docs/src/advanced/duckdb.md index 1105372f..c90b53a5 100644 --- a/docs/src/advanced/duckdb.md +++ b/docs/src/advanced/duckdb.md @@ -1,9 +1,13 @@ `TiPg` can serve tables, views and files (GeoParquet, or any format supported by DuckDB's `ST_Read`) through [DuckDB](https://duckdb.org) and its [spatial extension](https://duckdb.org/docs/extensions/spatial/overview), alongside or instead of PostgreSQL/PostGIS. +!!! warning "Experimental" + + The DuckDB backend is **experimental**: its API, settings and behavior may change in future releases, without following the deprecation policy of the rest of `TiPg`. Please report any issue or feedback on [GitHub](https://github.com/developmentseed/tipg/issues). + !!! note - The DuckDB backend is not used by the default `tipg.main:app` application: you need to create your own application (see examples below). + The DuckDB backend is not used by the default `tipg.main:app` application: you need to create your own application (see examples below, and the [example application](#example-application)). ### Install @@ -97,6 +101,25 @@ TIPG_DUCKDB_SOURCES='{"countries": "countries.parquet"}' \ uvicorn app:app --port 8000 ``` +### Example application + +A complete DuckDB application can be found in [`examples/duckdb`](https://github.com/developmentseed/tipg/tree/main/examples/duckdb): + +- [`main.py`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/main.py): a DuckDB version of `tipg.main:app` (CORS, cache-control and compression middlewares, viewer extension, `/healthz` endpoint), configured with the `TIPG_` and `TIPG_DUCKDB_` environment variables. +- [`build_source_coop_db.py`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/build_source_coop_db.py): a script creating a DuckDB database with one view per [Source Cooperative](https://source.coop) GeoParquet file listed in [`source_coop_geoparquet.md`](https://github.com/developmentseed/tipg/blob/main/examples/duckdb/source_coop_geoparquet.md). Views read the remote files directly (no data is copied). +- `source_coop.duckdb`: the database created by the script. + +```bash +cd examples/duckdb + +# Serve a remote GeoParquet file +TIPG_DUCKDB_SOURCES='{"buildings": "https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' \ +uv run --extra duckdb --group server uvicorn main:app + +# Serve the Source Cooperative GeoParquet files +TIPG_DUCKDB_DATABASE=source_coop.duckdb uv run --extra duckdb --group server uvicorn main:app +``` + ### PostgreSQL and DuckDB in a single application Each collection carries its own connection (`PgCollection.connection_pool`, `DuckCollection.connection`), so collections from both backends can be served by the same endpoints. Merge the collections from both backends in your `register_collection_catalog` function: diff --git a/examples/duckdb/.env-example b/examples/duckdb/.env-example new file mode 100644 index 00000000..a8509b7a --- /dev/null +++ b/examples/duckdb/.env-example @@ -0,0 +1,3 @@ +# TIPG_DUCKDB_DATABASE="source_coop.duckdb" +TIPG_DUCKDB_SOURCES='{"buildings": "https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet"}' +TIPG_DEFAULT_MINZOOM=11 \ No newline at end of file diff --git a/tipg/duck/__init__.py b/tipg/duck/__init__.py index 966cb31a..43d04249 100644 --- a/tipg/duck/__init__.py +++ b/tipg/duck/__init__.py @@ -1,5 +1,8 @@ """tipg.duck package: DuckDB integration for TiPg.""" +import warnings + +from ..errors import TipgExperimentalWarning from .collections import DuckCollection, duck_get_collection_index from .database import close_duckdb_connection, connect_to_duckdb, create_connection from .settings import DuckDBCatalogSettings, DuckDBSettings @@ -13,3 +16,10 @@ "DuckDBCatalogSettings", "DuckDBSettings", ] + + +warnings.warn( + "This module is experimental, its contents are subject to change and deprecation.", + category=TipgExperimentalWarning, + stacklevel=2, +) diff --git a/tipg/errors.py b/tipg/errors.py index a6ef9e1c..6b4fb200 100644 --- a/tipg/errors.py +++ b/tipg/errors.py @@ -68,6 +68,10 @@ class MissingCollectionCatalog(TiPgError): """`collection_catalog` not registered in the application state.""" +class TipgExperimentalWarning(UserWarning): + """Warning for experimental features in TiPg.""" + + DEFAULT_STATUS_CODES = { NotFound: status.HTTP_404_NOT_FOUND, InvalidBBox: status.HTTP_422_UNPROCESSABLE_CONTENT, From 8667ac45e1c00a0d137aae18f0ff2cd5c5b517a8 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 13 Nov 2026 11:21:21 +0100 Subject: [PATCH 21/22] chore: change viewer in example --- examples/duckdb/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/duckdb/main.py b/examples/duckdb/main.py index 869e7494..476f55d9 100644 --- a/examples/duckdb/main.py +++ b/examples/duckdb/main.py @@ -80,7 +80,7 @@ async def lifespan(app: FastAPI): ogc_api = Endpoints( title=settings.name, - with_tiles_viewer=settings.add_tiles_viewer, + with_tiles_viewer=False, extensions=[viewerExtension()], ) app.include_router(ogc_api.router) From 67d7710b0ec5b07b5302f403fefd756d07d5bcd1 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 13 Nov 2026 12:30:16 +0100 Subject: [PATCH 22/22] chore: add example for external dataset --- examples/duckdb/main_external.py | 169 +++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 examples/duckdb/main_external.py diff --git a/examples/duckdb/main_external.py b/examples/duckdb/main_external.py new file mode 100644 index 00000000..ffa98c92 --- /dev/null +++ b/examples/duckdb/main_external.py @@ -0,0 +1,169 @@ +"""tipg app with DuckDB backend. + +uv run --group server uvicorn main_external:app + +http://127.0.0.1:8000/collections/external/items?url=https://data.source.coop/vida/google-microsoft-osm-open-buildings/geoparquet/by_country/country_iso=FRA/FRA.parquet + +""" + +import datetime +import hashlib +from contextlib import asynccontextmanager +from typing import Annotated, cast + +import duckdb +from fastapi import FastAPI, HTTPException, Path, Query, Request +from starlette.concurrency import run_in_threadpool +from starlette.middleware.cors import CORSMiddleware +from starlette_cramjam.middleware import CompressionMiddleware + +from tipg import __version__ as tipg_version +from tipg.collections import Catalog +from tipg.duck import ( + DuckDBCatalogSettings, + DuckDBSettings, + close_duckdb_connection, + connect_to_duckdb, + duck_get_collection_index, +) +from tipg.duck.collections import DuckCollection +from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from tipg.extensions.viewer import viewerExtension +from tipg.factory import Endpoints +from tipg.middleware import CacheControlMiddleware +from tipg.openapi import _update_openapi +from tipg.settings import APISettings + +settings = APISettings() + +duckdb_catalog_settings = DuckDBCatalogSettings() + + +ALLOWED_SCHEMES = ("http://", "https://", "s3://") + + +def _quote_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +async def CollectionParams( + request: Request, + collectionId: Annotated[str, Path(description="Collection identifier")], + url: Annotated[str, Query(description="URL of the Geoparquet dataset")], +) -> DuckCollection: + """Return Layer Object.""" + if collectionId != "external": + raise HTTPException(404, f"Collection '{collectionId}' not found.") + + # Don't let users read local files (`/etc/...`, `file://`) or arbitrary formats + if not url.startswith(ALLOWED_SCHEMES) or not url.lower().endswith( + (".parquet", ".geoparquet") + ): + raise HTTPException(422, "`url` must be a remote (geo)parquet file.") + + cache: dict[str, DuckCollection] = request.app.state.external_collections + if collection := cache.get(url): + return collection + + # dynamically create DuckCollection based on the URL: + # 1. register the file as a view in the in-memory database + # 2. let tipg's catalog builder introspect that single view + name = "ext_" + hashlib.sha256(url.encode()).hexdigest()[:16] + conn = request.app.state.duckdb_connection + view = f'CREATE OR REPLACE VIEW memory.main."{name}" AS SELECT * FROM read_parquet({_quote_literal(url)}' + try: + try: + # file row number as stable feature id (same as tipg's `sources`) + await run_in_threadpool(conn.execute, f"{view}, file_row_number = true)") + except duckdb.BinderException: + await run_in_threadpool(conn.execute, f"{view})") + except duckdb.Error as e: + raise HTTPException(422, f"Could not read {url}: {e}") from e + + collections = await duck_get_collection_index( + connection=conn, + settings=duckdb_catalog_settings.model_copy( + update={"schemas": ["main"], "tables": [f"main.{name}"]} + ), + ) + if not collections: + raise HTTPException(422, f"No spatial collection found in {url}.") + + # expose it as `external` so links point back to `/collections/external/...` + collection = cast( + DuckCollection, collections[0].model_copy(update={"id": collectionId}) + ) + cache[url] = collection + return collection + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + # Create Connection Pool + await connect_to_duckdb(app, settings=DuckDBSettings(database=None, sources={})) + + # cache of url -> DuckCollection + app.state.external_collections = {} + # empty catalog so `/collections` doesn't error + app.state.collection_catalog = Catalog( + collections={}, last_updated=datetime.datetime.now() + ) + + yield + + # Close the Connection Pool + await close_duckdb_connection(app) + + +app = FastAPI( + title=settings.name, + version=tipg_version, + openapi_url="/api", + docs_url="/api.html", + lifespan=lifespan, + root_path=settings.root_path, +) + +# Fix OpenAPI response header for OGC Common compatibility +_update_openapi(app) + +ogc_api = Endpoints( + title=settings.name, + with_tiles_viewer=False, + extensions=[viewerExtension()], + collection_dependency=CollectionParams, # type: ignore +) +app.include_router(ogc_api.router) + + +# Set all CORS enabled origins +if settings.cors_origins: + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["GET"], + allow_headers=["*"], + ) + +app.add_middleware( + CacheControlMiddleware, + cachecontrol=settings.cachecontrol, + exclude_path={r"^/healthz$", r"^/metrics$"}, +) +app.add_middleware(CompressionMiddleware, compression_level=6) + +add_exception_handlers(app, DEFAULT_STATUS_CODES) + + +@app.get( + "/healthz", + description="Health Check.", + summary="Health Check.", + operation_id="healthCheck", + tags=["Health Check"], +) +def ping(): + """Health check.""" + return {"ping": "pong!"}